Prevent db locking when multiple queries from quay are accessing same tables (PROJQUAY-8758)

.gitignore

@@ -6,6 +6,7 @@ terraform-key.json
 terraform.tfstate*
 ansible/context
 image-archive.tar
+sqlite3.tar
 execution-environment.tar
 quay-aioi
 mirror-registry*

ansible-runner/context/app/project/roles/mirror_appliance/tasks/autodetect-sqlite-archive.yaml

@@ -10,5 +10,5 @@
 
 - name: Load sqlite image if sqlite3.tar exists
   shell: 
-    cmd: podman image import --change 'ENV PATH=/opt/app-root/src/bin:/opt/app-root/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' --change 'ENV container=oci' --change 'ENTRYPOINT=["/usr/bin/sqlite3"]' - {{ sqlite_image }} < {{ quay_root }}/sqlite3.tar
+    cmd: podman image import --change 'ENV PATH=/opt/app-root/src/bin:/opt/app-root/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' --change 'ENV container=oci' --change 'USER=1001' --change 'ENTRYPOINT=["/usr/bin/sqlite3"]' - {{ sqlite_image }} < {{ quay_root }}/sqlite3.tar
   when: s.stat.exists and local_install == "false"

ansible-runner/context/app/project/roles/mirror_appliance/tasks/install-quay-service.yaml

@@ -147,6 +147,45 @@
     name: "{{ sqlite_storage }}"
   when: "not sqlite_storage.startswith('/')"
 
+- name: Check if sqlite cli image is loaded
+  command: podman inspect --type=image {{ sqlite_image }}
+  register: sqlite_cli
+  ignore_errors: yes
+
+- name: Fail if sqlite cli image is not found
+  fail:
+    msg: "The SQLite CLI image '{{ sqlite_image }}' is not loaded."
+  when: sqlite_cli.rc != 0
+
+- name: Create quay database file with persistent WAL mode and correct permissions
+  block:
+    - name: Create DB file and set permissions with error messages
+      command: >
+        podman run --rm
+        -v {{ expanded_sqlite_storage }}:/sqlite:Z
+        --entrypoint /bin/sh
+        {{ sqlite_image }}
+        -c '
+          if ! sqlite3 /sqlite/quay_sqlite.db "PRAGMA journal_mode=WAL;"; then
+            echo "Failed to set WAL mode" >&2
+            exit 1
+          fi
+          if ! chown 1001:1001 /sqlite/quay_sqlite.db; then
+            echo "Failed to chown database file" >&2
+            exit 1
+          fi
+          if ! chmod 0664 /sqlite/quay_sqlite.db; then
+            echo "Failed to chmod database file" >&2
+            exit 1
+          fi
+        '
+      register: db_file_creation_result
+      failed_when: db_file_creation_result.rc != 0
+      changed_when: true
+      retries: 3
+      delay: 5
+
+
 - name: Start Quay service
   systemd:
     name: quay-app.service

ansible-runner/context/app/project/roles/mirror_appliance/tasks/main.yaml

@@ -13,6 +13,9 @@
 - name: Autodetect Image Archive
   include_tasks: autodetect-image-archive.yaml
 
+- name: Autodetect Sqlite Archive
+  include_tasks: autodetect-sqlite-archive.yaml
+
 - name: Install Quay Pod Service
   include_tasks: install-pod-service.yaml
 

ansible-runner/context/app/project/roles/mirror_appliance/templates/pg_to_sqlite.sh

@@ -34,6 +34,7 @@ pg_to_sqlite() {
         " "$input_file"
 
         echo "COMMIT;"
+        echo "PRAGMA journal_mode=WAL;"
     } > "$output_file"
 
     # Validate output file

cmd/install.go

@@ -134,6 +134,11 @@ func install() {
 	err = loadSSHKeys()
 	check(err)
 
+	var sqliteArchiveMountFlag string
+	// Load sqlite cli binary
+	sqliteArchiveMountFlag, err = loadSqliteCli()
+	check(err)
+
 	// Handle Image Archive Defaulting
 	var imageArchiveMountFlag string
 	if imageArchivePath == "" {
@@ -258,6 +263,7 @@ func install() {
 		`--workdir /runner/project `+
 		`--net host `+
 		imageArchiveMountFlag+ // optional image archive flag
+		sqliteArchiveMountFlag+
 		sslCertKeyFlag+ // optional ssl cert/key flag
 		` -v %s:/runner/env/ssh_key `+
 		`-e RUNNER_OMIT_EVENTS=False `+
@@ -268,8 +274,8 @@ func install() {
 		`--quiet `+
 		`--name ansible_runner_instance `+
 		fmt.Sprintf("%s ", eeImage)+
-		`ansible-playbook -i %s@%s, --private-key /runner/env/ssh_key -e "init_user=%s init_password=%s quay_image=%s quay_version=%s redis_image=%s pause_image=%s quay_hostname=%s local_install=%s quay_root=%s quay_storage=%s sqlite_storage=%s quay_cmd=%s" install_mirror_appliance.yml %s %s`,
-		sshKey, targetUsername, targetHostname, initUser, initPassword, quayImage, quayVersion, redisImage, pauseImage, quayHostname, strconv.FormatBool(isLocalInstall()), quayRoot, quayStorage, sqliteStorage, quayCmd, askBecomePassFlag, additionalArgs)
+		`ansible-playbook -i %s@%s, --private-key /runner/env/ssh_key -e "init_user=%s init_password=%s quay_image=%s quay_version=%s redis_image=%s sqlite_image=%s pause_image=%s quay_hostname=%s local_install=%s quay_root=%s quay_storage=%s sqlite_storage=%s quay_cmd=%s" install_mirror_appliance.yml %s %s`,
+		sshKey, targetUsername, targetHostname, initUser, initPassword, quayImage, quayVersion, redisImage, sqliteImage, pauseImage, quayHostname, strconv.FormatBool(isLocalInstall()), quayRoot, quayStorage, sqliteStorage, quayCmd, askBecomePassFlag, additionalArgs)
 
 	log.Debug("Running command: " + podmanCmd)
 	cmd := exec.Command("bash", "-c", podmanCmd)

cmd/utils.go

@@ -193,25 +193,28 @@ func loadSqliteCli() (string, error) {
 	if !pathExists(sqliteArchivePath) {
 		return "", errors.New("Could not find sqlite3.tar at " + sqliteArchivePath)
 	}
-	log.Info("Found sqlite3 cli binary at " + sqliteArchivePath)
-
-	sqliteArchiveMountFlag := fmt.Sprintf(" -v %s:/runner/sqlite3.tar", sqliteArchivePath)
-
-	if isLocalInstall() {
-		// Load sqlite3 as a podman image
-		log.Printf("Loading sqlite3 cli binary from sqlite3.tar")
-		statement := getImageMetadata("sqlite", sqliteImage, sqliteArchivePath)
-		sqliteImportCmd := exec.Command("/bin/bash", "-c", statement)
-		if verbose {
-			sqliteImportCmd.Stderr = os.Stderr
-			sqliteImportCmd.Stdout = os.Stdout
-		}
-		log.Debug("Importing sqlite3 cli binary with command: ", sqliteImportCmd)
-		err = sqliteImportCmd.Run()
-		if err != nil {
-			return "", err
+
+	var sqliteArchiveMountFlag string
+	if sqliteArchivePath != "" {
+		sqliteArchiveMountFlag = fmt.Sprintf(" -v %s:/runner/sqlite3.tar", sqliteArchivePath)
+		log.Info("Found sqlite archive at " + sqliteArchivePath)
+		if isLocalInstall() {
+			// Load sqlite3 as a podman image
+			log.Printf("Loading sqlite3 cli binary from sqlite3.tar")
+			statement := getImageMetadata("sqlite", sqliteImage, sqliteArchivePath)
+			sqliteImportCmd := exec.Command("/bin/bash", "-c", statement)
+			if verbose {
+				sqliteImportCmd.Stderr = os.Stderr
+				sqliteImportCmd.Stdout = os.Stdout
+			}
+			log.Debug("Importing sqlite3 cli binary with command: ", sqliteImportCmd)
+			err = sqliteImportCmd.Run()
+			if err != nil {
+				return "", err
+			}
 		}
 	}
+
 	log.Infof("Attempting to set SELinux rules on sqlite archive")
 	cmd := exec.Command("chcon", "-Rt", "svirt_sandbox_file_t", sqliteArchivePath)
 	if verbose {
@@ -240,6 +243,7 @@ func getImageMetadata(app, imageName, archivePath string) string {
 		statement = `/usr/bin/podman image import \
 					--change 'ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' \
 					--change 'ENV container=oci' \
+					--change 'USER=1001' \
 					--change 'ENTRYPOINT=["/usr/bin/sqlite3"]' \
 					- ` + imageName + ` < ` + archivePath
 	case "ansible":