Merge pull request #1 from DNXLabs/parallel-python

add parallel scripts

Author: brunodasilvalenga

scripts/parallel-script/README.md

@@ -0,0 +1,41 @@
+# Parallel sync/restore
+
+These scripts will help you to start a multi thread processment either for `aws s3 cp` and `mysql` restore. It's possible to adapt both scripts and use them for many multi process as you need.
+
+## Describing functions - parallel_db_restore.py
+
+- restore()
+    - The function responsible to run the mysql restore command.
+    - It will run a subprocess lib to call a syscall.
+    - Requires: a list of databases
+
+- list_dbs()
+    - This function will list the *.sql files in a directory 
+    - It will grab only the database name from the dump
+    - Requires:
+        - Run a dump database like: `mysql -N -e 'show databases' | while read dbname; do mysqldump --complete-insert --routines --triggers --single-transaction "$dbname" > /home/restore/"$dbname".sql; done`
+
+- main()
+    - pool = the number of threads running at the same time
+
+## Describing functions - parallel_sync.py
+
+- sync()
+    - The function responsible to run the `aws s3 cp` restore command.
+    - It will run a subprocess lib to call a syscall.
+    - Requires: a list of files (or folders) to sync
+
+- list_files()
+    - Create a file called my_files_list.txt with the files/folders list:
+        ```
+        file1
+        file2
+        file3
+        ```
+
+- main()
+    - pool = the number of threads running at the same time
+
+## Changing the scripts
+
+The scripts in this repo are examples. Feel free to change them to run your own parallel command. When changing the scripts don't forget to change the function to list whatever you want. 

scripts/parallel-script/parallel_db_restore.py

@@ -0,0 +1,33 @@
+import subprocess, os, time, sys
+import glob
+from multiprocessing import Pool
+
+users = []
+def restore(user):
+    my_host_dest="my_database.ap-southeast-2.rds.amazonaws.com"
+    db_admin="admin"
+    db_user="%s" % user
+    db_pass=" " #db_passw fill up this field
+    args = "/usr/bin/mysql -h %s -u %s -p%s < /home/restore/%s.sql" % (my_host_dest,db_admin,db_pass,db_user)
+    try:
+        subprocess.call('%s' % args, stderr=subprocess.PIPE, shell=True)
+    except Exception as error:
+        print(error)
+        sys.exit(1)
+
+def list_dbs():
+    dbs = glob.glob("/home/restore/*.sql")
+    for d in dbs:
+        db = d.split('.')[0].split('/')[4].split('_')[2]
+        users.append(db)
+    return users
+
+def main():
+    users = list_dbs()
+    start = time.time()
+    pool = Pool(20)
+    pool.map(restore, users)
+    print('Restore finished in %s seconds' % str(time.time() - start))
+
+if __name__ == '__main__':
+    main()

scripts/parallel-script/parallel_sync.py

@@ -0,0 +1,31 @@
+import subprocess, os, time, sys
+import glob
+from multiprocessing import Pool
+
+files = []
+def sync(file_name):
+    orig_bucket="s3://my-origin-bucket"  # origin bucket
+    dest_bucket="s3://my-destination-bucket" # destination bucket
+    args = "aws s3 cp %s/%s %s/%s >> ./files_finished.txt" % (orig_bucket,file_name,dest_bucket,file_name)
+    try:
+        proc = subprocess.call('%s' % args,
+                stderr=subprocess.PIPE, shell=True)
+    except Exception as error:
+        print error
+        sys.exit(1)
+
+def list_files():
+    with open("my_files_list.txt", "r") as file_list:
+        for f in file_list:
+            files.append(f.replace('\n', ' ').replace('\r', ''))
+        return files
+
+def main():
+    files = list_files()
+    start = time.time()
+    pool = Pool(15)
+    pool.map(sync, files)
+    print 'Sync finished in %s seconds' % str(time.time() - start)
+
+if __name__ == '__main__':
+    main()