Project

General

Profile

Helper Scripts » table_data_transfer.py

Enis Nuredini, 23.08.2023 11:18

 
1
import mysql.connector
2
import subprocess
3
import sys
4
import os
5
import getpass
6
import argparse
7

    
8
exit_flag = False
9
finish_flag = False
10

    
11
def get_global_exit():
12
    global exit_flag
13
    return exit_flag
14

    
15
def get_global_finish():
16
    global finish_flag
17
    return finish_flag
18

    
19
def restart_script():
20
    restart_choice = ""
21
    global exit_flag
22
    restart_choice = input("Do you want to restart the script from the beginning? (y/N): ").strip().lower()
23
    if restart_choice == "y":
24
        print("Restarting the script...")
25
        subprocess.run(['python3', 'table_data_transfer.py'])  # Restart the script
26
    else:
27
        print("Exiting script.")
28
        exit_flag = True
29
        sys.exit()
30

    
31
def truncate_and_import(tables, host, username, password, database, directory):
32
    connection = mysql.connector.connect(host=host,
33
                                 user=username,
34
                                 password=password,
35
                                 database=database)
36

    
37

    
38
    cursor = connection.cursor()
39
    for table in tables:
40
        print('Truncating target table: {}'.format(table))
41
        cursor.execute('TRUNCATE TABLE {}'.format(table))
42

    
43
        filename = '{}.sql'.format(table)
44
        filepath = os.path.join(directory, filename)
45
        print('Importing data to target table: {}'.format(table))
46
        with open(filepath) as sql_file:
47
            sql_query = sql_file.read()
48
            subprocess.run(['mysql', '-h', host, '-u', username, '-p'+password, database], input=sql_query, universal_newlines=True)
49

    
50
    cursor.close()
51
    connection.commit()
52
    connection.close()
53

    
54

    
55

    
56
def dump(tables, host, username, password, database, directory):
57
    try:
58
        for table in tables:
59
            try:
60
                filename = '{}.sql'.format(table)
61
                filepath = os.path.join(directory, filename)
62
                print('Dumping data from table: {}'.format(table))
63
                with open(filepath, 'w') as f:
64
                    process = subprocess.Popen(['mysqldump', '-h', host, '-u', username, '-p'+password, database, table], stdout=f, stderr=subprocess.PIPE, universal_newlines=True)
65
                    _, stderr_output = process.communicate()
66

    
67
                if stderr_output and "Access denied for user" in stderr_output:
68
                    print(f"Error: Access denied. Check your MySQL credentials. Password or db name ({database}) is wrong.")
69
                    restart_script()
70
                elif stderr_output and f'Couldn\'t find table: "{table}"' in stderr_output:
71
                    print(f"Error: Table '{database}.{table}' doesn't exist.")
72
                    restart_script()
73
                else:
74
                    print("Source dump completed successfully.")
75

    
76
            except mysql.connector.Error as e:
77
                if e.errno == mysql.connector.errorcode.ER_NO_SUCH_TABLE:
78
                    print("Error: Table '{}' doesn't exist.".format(table))
79
                else:
80
                    print("An mysql error occurred during source data dump:", e)
81
                restart_script()
82

    
83
    except subprocess.CalledProcessError as e:
84
        print("An error occurred during source data dump:", e)
85
        restart_script()
86
            
87
def dump_target(tables, host, username, password, database, directory):
88
    try:
89
        for table in tables:
90
            try:
91
                print('Dumping target data from table: {}'.format(table))
92
                file_number = 0
93

    
94
                filename = 'target_{}.sql'.format(table)
95
                filepath = os.path.join(directory, filename)
96

    
97
                while os.path.exists(filepath):
98
                    file_number += 1
99
                    filename = 'target_{}_{}.sql'.format(table, file_number)
100
                    filepath = os.path.join(directory, filename)
101

    
102
                with open(filepath, 'w') as f:
103
                    process = subprocess.Popen(['mysqldump', '-h', host, '-u', username, '-p'+password, database, table], stdout=f, stderr=subprocess.PIPE, universal_newlines=True)
104
                    _, stderr_output = process.communicate()
105

    
106
                if stderr_output and "Access denied for user" in stderr_output:
107
                    print(f"Error: Access denied. Check your MySQL credentials. Password or db name ({database}) is wrong.")
108
                    restart_script()
109
                elif stderr_output and f'Couldn\'t find table: "{table}"' in stderr_output:
110
                    print(f"Error: Table '{database}.{table}' doesn't exist.")
111
                    restart_script()
112
                else:
113
                    print("Target dump completed successfully.")
114

    
115
            except mysql.connector.Error as e:
116
                if e.errno == mysql.connector.errorcode.ER_NO_SUCH_TABLE:
117
                    print("Error: Table '{}' doesn't exist.".format(table))
118
                else:
119
                    print("An error occurred during data dump:", e)
120

    
121
                restart_script()
122

    
123
    except subprocess.CalledProcessError as e:
124
        print("An error occurred during target data dump:", e)
125
        restart_script()
126

    
127
def get_table_names():
128
    table_names = input("Enter comma-separated table names for data transfer: ").strip()
129
    table_names = [name.strip() for name in table_names.split(",")]
130
    clean_table_names = ",".join(table_names)
131
    return table_names
132

    
133
if __name__ == "__main__":
134

    
135
    directory = '/var/www/transfered_tables'    
136
    # Create the directory if it doesn't exist
137
    if not os.path.exists(directory):
138
        os.makedirs(directory)
139

    
140
    print("SQL dump destination: " + directory)
141
    # Dump from server 1
142
    dump_host = 'wdb16.math.uzh.ch'
143
    dump_user = 'enured'
144
    dump_database = input("Source DB name: ")
145
    dump_password = getpass.getpass('Enter database password for ' + dump_host + '.' + dump_database + ', user: ' + dump_user)
146

    
147
    # Import to server 2
148
    import_host = 'wdb22.math.uzh.ch'
149
    import_user = 'import_user'
150
    import_database = input("Target DB name: ")
151
    import_password = getpass.getpass('Enter database password for ' + import_host + '.' + import_database + ', user: ' + import_user)
152

    
153
    tables = get_table_names() # list of table names
154

    
155
    finish_flag = get_global_finish()
156
    exit_flag = get_global_exit()
157

    
158
    if not exit_flag:
159
        print("----------------------\nStarting dump source\n----------------------")
160
        dump(tables, dump_host, dump_user, dump_password, dump_database, directory)
161
        if not exit_flag:
162
            print("----------------------\nStarting dump target\n----------------------")
163
            dump_target(tables, import_host, import_user, import_password, import_database, directory)
164
            if not exit_flag and not finish_flag:
165
                print("----------------------\nStart truncate and import\n-----------------------")
166
                truncate_and_import(tables, import_host, import_user, import_password, import_database, directory)
167
                exit_flag = True
168

    
169
    dump_database = ""
170
    import_database = ""
(3-3/4)