chore: dev from dump improvement

.env.from-dump

@@ -1,2 +1,4 @@
 IGNORE_SEEDING=true
 DONT_EXECUTE_TRIGGERS=true
+FROM_DUMP=true
+SKIP_CACHE_REBUILD=true

.gitignore

@@ -39,3 +39,4 @@ prisma-local/seeds/productionTxs.csv
 paybutton-config.json
 
 dump.sql*
+.forever

Makefile

@@ -37,6 +37,18 @@ dev-from-dump:
 	$(touch_local_env)
 	docker compose -f docker-compose-from-dump.yml --env-file .env --env-file .env.local --env-file .env.from-dump up --build -d
 
+stop-dev-from-dump:
+	docker compose -f docker-compose-from-dump.yml --env-file .env --env-file .env.local --env-file .env.from-dump down
+
+reset-dev-from-dump:
+	make stop-dev-from-dump && make dev-from-dump
+
+reset-dev-from-dump-keep-db:
+	docker compose -f docker-compose-from-dump.yml --env-file .env --env-file .env.local --env-file .env.from-dump up --build -d --force-recreate --no-deps paybutton
+
+nuke-dev-from-dump:
+	docker compose -f docker-compose-from-dump.yml --env-file .env --env-file .env.local --env-file .env.from-dump down -v
+
 logs-dev:
 	docker logs -f paybutton-dev
 

docker-compose-from-dump.yml

@@ -42,6 +42,7 @@ services:
       - ./scripts/db/init-supertokens-db.sql:/home/mysql/raw_entrypoint/3-init-supertokens-db.sql
       - ./scripts/db/prisma-shadow-db-fix.sql:/home/mysql/raw_entrypoint/4-prisma-shadow-db-fix.sql
       - ./dump.sql:/home/mysql/raw_entrypoint/5-dump.sql
+      - paybutton-dump-db-data:/var/lib/mysql
 
   users-service:
     container_name: paybutton-users-service
@@ -69,3 +70,6 @@ services:
       - ./redis:/data/redis:z
       - ./scripts:/data/scripts
     command: sh -c "sed -i 's/\\r//g' ./scripts/redis-start.sh && sh ./scripts/redis-start.sh"
+
+volumes:
+  paybutton-dump-db-data:

redis/paymentCache.ts

@@ -31,9 +31,11 @@ export async function * getUserUncachedAddresses (userId: string): AsyncGenerato
 }
 
 export const getPaymentList = async (userId: string): Promise<Payment[]> => {
-  const uncachedAddressStream = getUserUncachedAddresses(userId)
-  for await (const address of uncachedAddressStream) {
-    void await CacheSet.addressCreation(address)
+  if (process.env.SKIP_CACHE_REBUILD === undefined) {
+    const uncachedAddressStream = getUserUncachedAddresses(userId)
+    for await (const address of uncachedAddressStream) {
+      void await CacheSet.addressCreation(address)
+    }
   }
   return await getCachedPaymentsForUser(userId)
 }
@@ -304,6 +306,7 @@ export const clearPaymentCacheForAddress = async (addressString: string): Promis
 }
 
 export const initPaymentCache = async (address: Address): Promise<boolean> => {
+  if (process.env.SKIP_CACHE_REBUILD !== undefined) return false
   const cachedKeys = await getCachedWeekKeysForAddress(address.address)
   if (cachedKeys.length === 0) {
     await CacheSet.addressCreation(address)
@@ -313,10 +316,12 @@ export const initPaymentCache = async (address: Address): Promise<boolean> => {
 }
 
 export async function * getPaymentStream (userId: string): AsyncGenerator<Payment> {
-  const uncachedAddressStream = getUserUncachedAddresses(userId)
-  for await (const address of uncachedAddressStream) {
-    console.log('[CACHE]: Creating cache for address', address.address)
-    await CacheSet.addressCreation(address)
+  if (process.env.SKIP_CACHE_REBUILD === undefined) {
+    const uncachedAddressStream = getUserUncachedAddresses(userId)
+    for await (const address of uncachedAddressStream) {
+      console.log('[CACHE]: Creating cache for address', address.address)
+      await CacheSet.addressCreation(address)
+    }
   }
   const userButtonIds: string[] = (await fetchPaybuttonArrayByUserId(userId))
     .map(p => p.id)

scripts/db-entrypoint.sh

@@ -10,11 +10,25 @@ sed_vars () {
 cd /home/mysql/raw_entrypoint || exit
 
 for file in *.sql; do
-    sed_vars "$file"
+    if [[ "$file" == *dump* ]]; then
+      echo "Linking dump file: $file"
+      ln -s /home/mysql/raw_entrypoint/"$file" /home/mysql/entrypoint/"$file"
+    else
+        echo "Copying file $file after seding vars"
+        sed_vars "$file"
+    fi
 done
 
 cd ../entrypoint/ || exit
 
 for file in *.sql; do
-    mariadb -u root -p"$MYSQL_ROOT_PASSWORD" < "$file"
+    if [[ "$file" == *dump* ]]; then
+        filesize=$(stat -Lc%s "$file" 2>/dev/null || stat -Lf%z "$file" 2>/dev/null)
+        echo "Importing $file ($(numfmt --to=iec $filesize)) ..."
+        pv "$file" | mariadb -u root -p"$MYSQL_ROOT_PASSWORD"
+    else
+        echo "Importing $file ..."
+        mariadb -u root -p"$MYSQL_ROOT_PASSWORD" < "$file"
+    fi
+    echo "Done importing $file"
 done

scripts/db/Dockerfile

@@ -2,8 +2,8 @@ FROM mariadb:latest
 
 RUN mkhomedir_helper mysql
 
-RUN apt-get update || echo && \
-      apt-get install -y gettext || echo
+RUN apt-get update || true && \
+      apt-get install -y gettext pv || true
 
 RUN mkdir -p /home/mysql/raw_entrypoint && \
     mkdir -p /home/mysql/entrypoint

scripts/paybutton-server-start.sh

@@ -17,17 +17,19 @@ logtime=$(date +%Y-%m-%d@%H:%M)
 [ -e logs/jobs.log ] && mv logs/jobs.log logs/history/jobs_"$logtime".log
 [ -e logs/ws-server.log ] && mv logs/ws-server.log logs/history/ws-server_"$logtime".log
 
-if [ "$ENVIRONMENT" = "production" ]; then
+if [ "$FROM_DUMP" = "true" ] || [ "$ENVIRONMENT" = "production" ]; then
     yarn prisma migrate deploy || exit 1
     yarn prisma generate || exit 1
-    pm2 start yarn --time --interpreter ash --name jobs --output logs/jobs.log --error logs/jobs.log -- initJobs || exit 1
-    pm2 start yarn --time --interpreter ash --name WSServer --output logs/ws-server.log --error logs/ws-server.log -- initWSServer || exit 1
-    pm2 start yarn --time --interpreter ash --name next --output logs/next.log --error logs/next.log -- prod || exit 1
 else
     yarn prisma migrate dev || exit 1
     yarn prisma db seed || exit 1
-    pm2 start yarn --time --interpreter ash --name jobs --output logs/jobs.log --error logs/jobs.log -- initJobs || exit 1
-    pm2 start yarn --time --interpreter ash --name WSServer --output logs/ws-server.log --error logs/ws-server.log -- initWSServer || exit 1
+fi
+
+pm2 start yarn --time --interpreter ash --name jobs --output logs/jobs.log --error logs/jobs.log -- initJobs || exit 1
+pm2 start yarn --time --interpreter ash --name WSServer --output logs/ws-server.log --error logs/ws-server.log -- initWSServer || exit 1
+if [ "$ENVIRONMENT" = "production" ]; then
+    pm2 start yarn --time --interpreter ash --name next --output logs/next.log --error logs/next.log -- prod || exit 1
+else
     pm2 start yarn --time --interpreter ash --name next --output logs/next.log --error logs/next.log -- dev || exit 1
 fi
 pm2 logs next