peuhsdf commited on
Commit
a19d888
·
1 Parent(s): 13ca8e6
Project/app/hkjc/soccer/result/import/.ipynb_checkpoints/import-backup-checkpoint.ipynb ADDED
@@ -0,0 +1,229 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": 1,
6
+ "id": "c0a06deb-cb7b-4372-a7fd-675380b30353",
7
+ "metadata": {},
8
+ "outputs": [],
9
+ "source": [
10
+ "%run /data/Project/scripts/common/references.ipynb"
11
+ ]
12
+ },
13
+ {
14
+ "cell_type": "code",
15
+ "execution_count": 48,
16
+ "id": "840f09be-4491-43d6-bca9-7885b4e0befa",
17
+ "metadata": {},
18
+ "outputs": [
19
+ {
20
+ "data": {
21
+ "application/json": {
22
+ "ascii": false,
23
+ "bar_format": null,
24
+ "colour": null,
25
+ "elapsed": 0.004201173782348633,
26
+ "initial": 0,
27
+ "n": 0,
28
+ "ncols": null,
29
+ "nrows": null,
30
+ "postfix": null,
31
+ "prefix": "",
32
+ "rate": null,
33
+ "total": 14325,
34
+ "unit": "it",
35
+ "unit_divisor": 1000,
36
+ "unit_scale": false
37
+ },
38
+ "application/vnd.jupyter.widget-view+json": {
39
+ "model_id": "906cc603a7a24ded9ef7e8f6a3f5c2e7",
40
+ "version_major": 2,
41
+ "version_minor": 0
42
+ },
43
+ "text/plain": [
44
+ " 0%| | 0/14325 [00:00<?, ?it/s]"
45
+ ]
46
+ },
47
+ "metadata": {},
48
+ "output_type": "display_data"
49
+ },
50
+ {
51
+ "name": "stdout",
52
+ "output_type": "stream",
53
+ "text": [
54
+ "INSERT OR IGNORE INTO '_20030802_1__tournaments_1_' ('name','matchescount') VALUES ('SearchMatches','7')\n"
55
+ ]
56
+ },
57
+ {
58
+ "ename": "OperationalError",
59
+ "evalue": "database is locked",
60
+ "output_type": "error",
61
+ "traceback": [
62
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
63
+ "\u001b[0;31mOperationalError\u001b[0m Traceback (most recent call last)",
64
+ "Cell \u001b[0;32mIn[48], line 62\u001b[0m\n\u001b[1;32m 59\u001b[0m progress\u001b[38;5;241m.\u001b[39mupdate(\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m 60\u001b[0m \u001b[38;5;28;01mbreak\u001b[39;00m\n\u001b[0;32m---> 62\u001b[0m \u001b[43mmain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 63\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdone\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
65
+ "Cell \u001b[0;32mIn[48], line 54\u001b[0m, in \u001b[0;36mmain\u001b[0;34m()\u001b[0m\n\u001b[1;32m 52\u001b[0m values\u001b[38;5;241m.\u001b[39mappend(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mvalue\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 53\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m---> 54\u001b[0m \u001b[43minsert_row\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\u001b[43mtblname\u001b[49m\u001b[43m,\u001b[49m\u001b[43mcolnames\u001b[49m\u001b[43m,\u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 55\u001b[0m colnames \u001b[38;5;241m=\u001b[39m [\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mcolname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m 56\u001b[0m values \u001b[38;5;241m=\u001b[39m [\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mvalue\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n",
66
+ "Cell \u001b[0;32mIn[48], line 24\u001b[0m, in \u001b[0;36minsert_row\u001b[0;34m(conn, tblname, colnames, values)\u001b[0m\n\u001b[1;32m 22\u001b[0m create_table(conn,tblname)\n\u001b[1;32m 23\u001b[0m add_columns(conn,tblname,colnames)\n\u001b[0;32m---> 24\u001b[0m \u001b[43minsert_into\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\u001b[43mtblname\u001b[49m\u001b[43m,\u001b[49m\u001b[43mcolnames\u001b[49m\u001b[43m,\u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m)\u001b[49m\n",
67
+ "Cell \u001b[0;32mIn[48], line 28\u001b[0m, in \u001b[0;36minsert_into\u001b[0;34m(conn, tblname, colnames, values)\u001b[0m\n\u001b[1;32m 26\u001b[0m sql \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mINSERT OR IGNORE INTO \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtblname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m (\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(colnames)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m) VALUES (\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(values)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m)\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 27\u001b[0m \u001b[38;5;28mprint\u001b[39m(sql)\n\u001b[0;32m---> 28\u001b[0m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcursor\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[43msql\u001b[49m\u001b[43m)\u001b[49m\n",
68
+ "\u001b[0;31mOperationalError\u001b[0m: database is locked"
69
+ ]
70
+ }
71
+ ],
72
+ "source": [
73
+ "def read_dict(file):\n",
74
+ " return Json.deserialize(File.read(file))\n",
75
+ "def edit_tblname(tblname):\n",
76
+ " return tblname.replace(\"/\",\"_\").replace(\"[\",\"_\").replace(\"]\",\"_\")\n",
77
+ "def create_table(conn,tblname):\n",
78
+ " sql = f\"CREATE TABLE IF NOT EXISTS '{tblname}' ('ID' INT PRIARY KEY UNIQUE)\"\n",
79
+ " conn.cursor().execute(sql)\n",
80
+ "def add_column(conn,tblname,colname):\n",
81
+ " sql = f\"ALTER TABLE {tblname} ADD COLUMN {colname} TEXT;\"\n",
82
+ " conn.cursor().execute(sql)\n",
83
+ "def add_columns(conn,tblname,colnames):\n",
84
+ " sql = f\"PRAGMA TABLE_INFO({tblname})\"\n",
85
+ " cursor = conn.cursor()\n",
86
+ " cursor.execute(sql)\n",
87
+ " cols = []\n",
88
+ " for row in cursor:\n",
89
+ " cols.append(f\"'{row[1]}'\")\n",
90
+ " for colname in colnames:\n",
91
+ " if not colname in cols:\n",
92
+ " add_column(conn,tblname,colname)\n",
93
+ "def insert_row(conn,tblname,colnames,values):\n",
94
+ " create_table(conn,tblname)\n",
95
+ " add_columns(conn,tblname,colnames)\n",
96
+ " insert_into(conn,tblname,colnames,values)\n",
97
+ "def insert_into(conn,tblname,colnames,values):\n",
98
+ " sql = f\"INSERT OR IGNORE INTO '{tblname}' ({','.join(colnames)}) VALUES ({','.join(values)})\"\n",
99
+ " print(sql)\n",
100
+ " conn.cursor().execute(sql)\n",
101
+ "\n",
102
+ "def main():\n",
103
+ " RESULT_PATH_ROOT = \"/data/Dataset/hkjc/soccer/result/path\"\n",
104
+ " with closing(sqlite3.connect(\"test.db\")) as conn:\n",
105
+ " files = File.get_files(RESULT_PATH_ROOT)\n",
106
+ " progress = Progress(len(files))\n",
107
+ " for file in files:\n",
108
+ " dic = read_dict(file)\n",
109
+ " last_tblname = None\n",
110
+ " for key in dic:\n",
111
+ " value = dic[key]\n",
112
+ " if type(value)==bool:\n",
113
+ " value = str(value)\n",
114
+ " tblname = os.path.dirname(key)\n",
115
+ " tblname = edit_tblname(tblname)\n",
116
+ " colname = os.path.basename(key)\n",
117
+ " colname = edit_tblname(colname)\n",
118
+ " # print(parent,name,value)\n",
119
+ " if last_tblname == None:\n",
120
+ " colnames = [f\"'{colname}'\"]\n",
121
+ " values = [f\"'{value}'\"]\n",
122
+ " elif tblname == last_tblname: \n",
123
+ " colnames.append(f\"'{colname}'\")\n",
124
+ " values.append(f\"'{value}'\")\n",
125
+ " else:\n",
126
+ " insert_row(conn,tblname,colnames,values)\n",
127
+ " colnames = [f\"'{colname}'\"]\n",
128
+ " values = [f\"'{value}'\"]\n",
129
+ " last_tblname = tblname\n",
130
+ " insert_row(conn,tblname,colnames,values)\n",
131
+ " progress.update(1)\n",
132
+ " break\n",
133
+ "\n",
134
+ "main()\n",
135
+ "print(\"done\")"
136
+ ]
137
+ },
138
+ {
139
+ "cell_type": "code",
140
+ "execution_count": 33,
141
+ "id": "d261f797-7fb4-407a-857e-430f534bfa32",
142
+ "metadata": {},
143
+ "outputs": [
144
+ {
145
+ "name": "stdout",
146
+ "output_type": "stream",
147
+ "text": [
148
+ "['T', 'T2']\n",
149
+ "['C']\n",
150
+ "['C', 'C2']\n"
151
+ ]
152
+ }
153
+ ],
154
+ "source": [
155
+ "!rm test.db -r\n",
156
+ "\n",
157
+ "conn = sqlite3.connect(\"test.db\")\n",
158
+ "\n",
159
+ "sql = \"CREATE TABLE IF NOT EXISTS 'T' ('C' INT PRIMARY KEY UNIQUE);\"\n",
160
+ "cursor = conn.cursor()\n",
161
+ "cursor.execute(sql)\n",
162
+ "cursor.close()\n",
163
+ "\n",
164
+ "sql = \"CREATE TABLE IF NOT EXISTS 'T2' ('C2' INT PRIMARY KEY UNIQUE);\"\n",
165
+ "cursor = conn.cursor()\n",
166
+ "cursor.execute(sql)\n",
167
+ "cursor.close()\n",
168
+ "\n",
169
+ "sql = f\"SELECT TBL_NAME FROM SQLITE_MASTER GROUP BY TBL_NAME;\"\n",
170
+ "cursor = conn.cursor()\n",
171
+ "cursor.execute(sql)\n",
172
+ "tblnames = [row[0] for row in cursor]\n",
173
+ "print(tblnames)\n",
174
+ "cursor.close()\n",
175
+ "\n",
176
+ "\n",
177
+ "colnames = []\n",
178
+ "for tblname in tblnames:\n",
179
+ " sql = f\"PRAGMA TABLE_INFO({tblname})\"\n",
180
+ " cursor = conn.cursor()\n",
181
+ " cursor.execute(sql)\n",
182
+ " for row in cursor:\n",
183
+ " colnames.append(row[1])\n",
184
+ " print(colnames)\n",
185
+ " cursor.close()\n",
186
+ " \n",
187
+ "\n",
188
+ "conn.close()"
189
+ ]
190
+ },
191
+ {
192
+ "cell_type": "code",
193
+ "execution_count": null,
194
+ "id": "930632eb-86c7-4d63-b9ae-b1df7d3d43ef",
195
+ "metadata": {},
196
+ "outputs": [],
197
+ "source": []
198
+ },
199
+ {
200
+ "cell_type": "code",
201
+ "execution_count": null,
202
+ "id": "5d8f58f3-5a9f-4cbd-8930-d365a4ec4819",
203
+ "metadata": {},
204
+ "outputs": [],
205
+ "source": []
206
+ }
207
+ ],
208
+ "metadata": {
209
+ "kernelspec": {
210
+ "display_name": "Python 3 (ipykernel)",
211
+ "language": "python",
212
+ "name": "python3"
213
+ },
214
+ "language_info": {
215
+ "codemirror_mode": {
216
+ "name": "ipython",
217
+ "version": 3
218
+ },
219
+ "file_extension": ".py",
220
+ "mimetype": "text/x-python",
221
+ "name": "python",
222
+ "nbconvert_exporter": "python",
223
+ "pygments_lexer": "ipython3",
224
+ "version": "3.9.5"
225
+ }
226
+ },
227
+ "nbformat": 4,
228
+ "nbformat_minor": 5
229
+ }
Project/app/hkjc/soccer/result/import/.ipynb_checkpoints/import-checkpoint.ipynb ADDED
@@ -0,0 +1,229 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": 1,
6
+ "id": "c0a06deb-cb7b-4372-a7fd-675380b30353",
7
+ "metadata": {},
8
+ "outputs": [],
9
+ "source": [
10
+ "%run /data/Project/scripts/common/references.ipynb"
11
+ ]
12
+ },
13
+ {
14
+ "cell_type": "code",
15
+ "execution_count": 48,
16
+ "id": "840f09be-4491-43d6-bca9-7885b4e0befa",
17
+ "metadata": {},
18
+ "outputs": [
19
+ {
20
+ "data": {
21
+ "application/json": {
22
+ "ascii": false,
23
+ "bar_format": null,
24
+ "colour": null,
25
+ "elapsed": 0.004201173782348633,
26
+ "initial": 0,
27
+ "n": 0,
28
+ "ncols": null,
29
+ "nrows": null,
30
+ "postfix": null,
31
+ "prefix": "",
32
+ "rate": null,
33
+ "total": 14325,
34
+ "unit": "it",
35
+ "unit_divisor": 1000,
36
+ "unit_scale": false
37
+ },
38
+ "application/vnd.jupyter.widget-view+json": {
39
+ "model_id": "906cc603a7a24ded9ef7e8f6a3f5c2e7",
40
+ "version_major": 2,
41
+ "version_minor": 0
42
+ },
43
+ "text/plain": [
44
+ " 0%| | 0/14325 [00:00<?, ?it/s]"
45
+ ]
46
+ },
47
+ "metadata": {},
48
+ "output_type": "display_data"
49
+ },
50
+ {
51
+ "name": "stdout",
52
+ "output_type": "stream",
53
+ "text": [
54
+ "INSERT OR IGNORE INTO '_20030802_1__tournaments_1_' ('name','matchescount') VALUES ('SearchMatches','7')\n"
55
+ ]
56
+ },
57
+ {
58
+ "ename": "OperationalError",
59
+ "evalue": "database is locked",
60
+ "output_type": "error",
61
+ "traceback": [
62
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
63
+ "\u001b[0;31mOperationalError\u001b[0m Traceback (most recent call last)",
64
+ "Cell \u001b[0;32mIn[48], line 62\u001b[0m\n\u001b[1;32m 59\u001b[0m progress\u001b[38;5;241m.\u001b[39mupdate(\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m 60\u001b[0m \u001b[38;5;28;01mbreak\u001b[39;00m\n\u001b[0;32m---> 62\u001b[0m \u001b[43mmain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 63\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdone\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
65
+ "Cell \u001b[0;32mIn[48], line 54\u001b[0m, in \u001b[0;36mmain\u001b[0;34m()\u001b[0m\n\u001b[1;32m 52\u001b[0m values\u001b[38;5;241m.\u001b[39mappend(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mvalue\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 53\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m---> 54\u001b[0m \u001b[43minsert_row\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\u001b[43mtblname\u001b[49m\u001b[43m,\u001b[49m\u001b[43mcolnames\u001b[49m\u001b[43m,\u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 55\u001b[0m colnames \u001b[38;5;241m=\u001b[39m [\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mcolname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m 56\u001b[0m values \u001b[38;5;241m=\u001b[39m [\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mvalue\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n",
66
+ "Cell \u001b[0;32mIn[48], line 24\u001b[0m, in \u001b[0;36minsert_row\u001b[0;34m(conn, tblname, colnames, values)\u001b[0m\n\u001b[1;32m 22\u001b[0m create_table(conn,tblname)\n\u001b[1;32m 23\u001b[0m add_columns(conn,tblname,colnames)\n\u001b[0;32m---> 24\u001b[0m \u001b[43minsert_into\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\u001b[43mtblname\u001b[49m\u001b[43m,\u001b[49m\u001b[43mcolnames\u001b[49m\u001b[43m,\u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m)\u001b[49m\n",
67
+ "Cell \u001b[0;32mIn[48], line 28\u001b[0m, in \u001b[0;36minsert_into\u001b[0;34m(conn, tblname, colnames, values)\u001b[0m\n\u001b[1;32m 26\u001b[0m sql \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mINSERT OR IGNORE INTO \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtblname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m (\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(colnames)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m) VALUES (\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(values)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m)\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 27\u001b[0m \u001b[38;5;28mprint\u001b[39m(sql)\n\u001b[0;32m---> 28\u001b[0m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcursor\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[43msql\u001b[49m\u001b[43m)\u001b[49m\n",
68
+ "\u001b[0;31mOperationalError\u001b[0m: database is locked"
69
+ ]
70
+ }
71
+ ],
72
+ "source": [
73
+ "def read_dict(file):\n",
74
+ " return Json.deserialize(File.read(file))\n",
75
+ "def edit_tblname(tblname):\n",
76
+ " return tblname.replace(\"/\",\"_\").replace(\"[\",\"_\").replace(\"]\",\"_\")\n",
77
+ "def create_table(conn,tblname):\n",
78
+ " sql = f\"CREATE TABLE IF NOT EXISTS '{tblname}' ('ID' INT PRIARY KEY UNIQUE)\"\n",
79
+ " conn.cursor().execute(sql)\n",
80
+ "def add_column(conn,tblname,colname):\n",
81
+ " sql = f\"ALTER TABLE {tblname} ADD COLUMN {colname} TEXT;\"\n",
82
+ " conn.cursor().execute(sql)\n",
83
+ "def add_columns(conn,tblname,colnames):\n",
84
+ " sql = f\"PRAGMA TABLE_INFO({tblname})\"\n",
85
+ " cursor = conn.cursor()\n",
86
+ " cursor.execute(sql)\n",
87
+ " cols = []\n",
88
+ " for row in cursor:\n",
89
+ " cols.append(f\"'{row[1]}'\")\n",
90
+ " for colname in colnames:\n",
91
+ " if not colname in cols:\n",
92
+ " add_column(conn,tblname,colname)\n",
93
+ "def insert_row(conn,tblname,colnames,values):\n",
94
+ " create_table(conn,tblname)\n",
95
+ " add_columns(conn,tblname,colnames)\n",
96
+ " insert_into(conn,tblname,colnames,values)\n",
97
+ "def insert_into(conn,tblname,colnames,values):\n",
98
+ " sql = f\"INSERT OR IGNORE INTO '{tblname}' ({','.join(colnames)}) VALUES ({','.join(values)})\"\n",
99
+ " print(sql)\n",
100
+ " conn.cursor().execute(sql)\n",
101
+ "\n",
102
+ "def main():\n",
103
+ " RESULT_PATH_ROOT = \"/data/Dataset/hkjc/soccer/result/path\"\n",
104
+ " with closing(sqlite3.connect(\"test.db\")) as conn:\n",
105
+ " files = File.get_files(RESULT_PATH_ROOT)\n",
106
+ " progress = Progress(len(files))\n",
107
+ " for file in files:\n",
108
+ " dic = read_dict(file)\n",
109
+ " last_tblname = None\n",
110
+ " for key in dic:\n",
111
+ " value = dic[key]\n",
112
+ " if type(value)==bool:\n",
113
+ " value = str(value)\n",
114
+ " tblname = os.path.dirname(key)\n",
115
+ " tblname = edit_tblname(tblname)\n",
116
+ " colname = os.path.basename(key)\n",
117
+ " colname = edit_tblname(colname)\n",
118
+ " # print(parent,name,value)\n",
119
+ " if last_tblname == None:\n",
120
+ " colnames = [f\"'{colname}'\"]\n",
121
+ " values = [f\"'{value}'\"]\n",
122
+ " elif tblname == last_tblname: \n",
123
+ " colnames.append(f\"'{colname}'\")\n",
124
+ " values.append(f\"'{value}'\")\n",
125
+ " else:\n",
126
+ " insert_row(conn,tblname,colnames,values)\n",
127
+ " colnames = [f\"'{colname}'\"]\n",
128
+ " values = [f\"'{value}'\"]\n",
129
+ " last_tblname = tblname\n",
130
+ " insert_row(conn,tblname,colnames,values)\n",
131
+ " progress.update(1)\n",
132
+ " break\n",
133
+ "\n",
134
+ "main()\n",
135
+ "print(\"done\")"
136
+ ]
137
+ },
138
+ {
139
+ "cell_type": "code",
140
+ "execution_count": 33,
141
+ "id": "d261f797-7fb4-407a-857e-430f534bfa32",
142
+ "metadata": {},
143
+ "outputs": [
144
+ {
145
+ "name": "stdout",
146
+ "output_type": "stream",
147
+ "text": [
148
+ "['T', 'T2']\n",
149
+ "['C']\n",
150
+ "['C', 'C2']\n"
151
+ ]
152
+ }
153
+ ],
154
+ "source": [
155
+ "!rm test.db -r\n",
156
+ "\n",
157
+ "conn = sqlite3.connect(\"test.db\")\n",
158
+ "\n",
159
+ "sql = \"CREATE TABLE IF NOT EXISTS 'T' ('C' INT PRIMARY KEY UNIQUE);\"\n",
160
+ "cursor = conn.cursor()\n",
161
+ "cursor.execute(sql)\n",
162
+ "cursor.close()\n",
163
+ "\n",
164
+ "sql = \"CREATE TABLE IF NOT EXISTS 'T2' ('C2' INT PRIMARY KEY UNIQUE);\"\n",
165
+ "cursor = conn.cursor()\n",
166
+ "cursor.execute(sql)\n",
167
+ "cursor.close()\n",
168
+ "\n",
169
+ "sql = f\"SELECT TBL_NAME FROM SQLITE_MASTER GROUP BY TBL_NAME;\"\n",
170
+ "cursor = conn.cursor()\n",
171
+ "cursor.execute(sql)\n",
172
+ "tblnames = [row[0] for row in cursor]\n",
173
+ "print(tblnames)\n",
174
+ "cursor.close()\n",
175
+ "\n",
176
+ "\n",
177
+ "colnames = []\n",
178
+ "for tblname in tblnames:\n",
179
+ " sql = f\"PRAGMA TABLE_INFO({tblname})\"\n",
180
+ " cursor = conn.cursor()\n",
181
+ " cursor.execute(sql)\n",
182
+ " for row in cursor:\n",
183
+ " colnames.append(row[1])\n",
184
+ " print(colnames)\n",
185
+ " cursor.close()\n",
186
+ " \n",
187
+ "\n",
188
+ "conn.close()"
189
+ ]
190
+ },
191
+ {
192
+ "cell_type": "code",
193
+ "execution_count": null,
194
+ "id": "930632eb-86c7-4d63-b9ae-b1df7d3d43ef",
195
+ "metadata": {},
196
+ "outputs": [],
197
+ "source": []
198
+ },
199
+ {
200
+ "cell_type": "code",
201
+ "execution_count": null,
202
+ "id": "5d8f58f3-5a9f-4cbd-8930-d365a4ec4819",
203
+ "metadata": {},
204
+ "outputs": [],
205
+ "source": []
206
+ }
207
+ ],
208
+ "metadata": {
209
+ "kernelspec": {
210
+ "display_name": "Python 3 (ipykernel)",
211
+ "language": "python",
212
+ "name": "python3"
213
+ },
214
+ "language_info": {
215
+ "codemirror_mode": {
216
+ "name": "ipython",
217
+ "version": 3
218
+ },
219
+ "file_extension": ".py",
220
+ "mimetype": "text/x-python",
221
+ "name": "python",
222
+ "nbconvert_exporter": "python",
223
+ "pygments_lexer": "ipython3",
224
+ "version": "3.9.5"
225
+ }
226
+ },
227
+ "nbformat": 4,
228
+ "nbformat_minor": 5
229
+ }
Project/app/hkjc/soccer/result/import/.ipynb_checkpoints/import-json-2-checkpoint.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Project/app/hkjc/soccer/result/import/.ipynb_checkpoints/import-json-checkpoint.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Project/app/hkjc/soccer/result/import/.ipynb_checkpoints/sqlite3helper-checkpoint.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Project/app/hkjc/soccer/result/import/import-backup.ipynb ADDED
@@ -0,0 +1,229 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": 1,
6
+ "id": "c0a06deb-cb7b-4372-a7fd-675380b30353",
7
+ "metadata": {},
8
+ "outputs": [],
9
+ "source": [
10
+ "%run /data/Project/scripts/common/references.ipynb"
11
+ ]
12
+ },
13
+ {
14
+ "cell_type": "code",
15
+ "execution_count": 48,
16
+ "id": "840f09be-4491-43d6-bca9-7885b4e0befa",
17
+ "metadata": {},
18
+ "outputs": [
19
+ {
20
+ "data": {
21
+ "application/json": {
22
+ "ascii": false,
23
+ "bar_format": null,
24
+ "colour": null,
25
+ "elapsed": 0.004201173782348633,
26
+ "initial": 0,
27
+ "n": 0,
28
+ "ncols": null,
29
+ "nrows": null,
30
+ "postfix": null,
31
+ "prefix": "",
32
+ "rate": null,
33
+ "total": 14325,
34
+ "unit": "it",
35
+ "unit_divisor": 1000,
36
+ "unit_scale": false
37
+ },
38
+ "application/vnd.jupyter.widget-view+json": {
39
+ "model_id": "906cc603a7a24ded9ef7e8f6a3f5c2e7",
40
+ "version_major": 2,
41
+ "version_minor": 0
42
+ },
43
+ "text/plain": [
44
+ " 0%| | 0/14325 [00:00<?, ?it/s]"
45
+ ]
46
+ },
47
+ "metadata": {},
48
+ "output_type": "display_data"
49
+ },
50
+ {
51
+ "name": "stdout",
52
+ "output_type": "stream",
53
+ "text": [
54
+ "INSERT OR IGNORE INTO '_20030802_1__tournaments_1_' ('name','matchescount') VALUES ('SearchMatches','7')\n"
55
+ ]
56
+ },
57
+ {
58
+ "ename": "OperationalError",
59
+ "evalue": "database is locked",
60
+ "output_type": "error",
61
+ "traceback": [
62
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
63
+ "\u001b[0;31mOperationalError\u001b[0m Traceback (most recent call last)",
64
+ "Cell \u001b[0;32mIn[48], line 62\u001b[0m\n\u001b[1;32m 59\u001b[0m progress\u001b[38;5;241m.\u001b[39mupdate(\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m 60\u001b[0m \u001b[38;5;28;01mbreak\u001b[39;00m\n\u001b[0;32m---> 62\u001b[0m \u001b[43mmain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 63\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdone\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
65
+ "Cell \u001b[0;32mIn[48], line 54\u001b[0m, in \u001b[0;36mmain\u001b[0;34m()\u001b[0m\n\u001b[1;32m 52\u001b[0m values\u001b[38;5;241m.\u001b[39mappend(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mvalue\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 53\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m---> 54\u001b[0m \u001b[43minsert_row\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\u001b[43mtblname\u001b[49m\u001b[43m,\u001b[49m\u001b[43mcolnames\u001b[49m\u001b[43m,\u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 55\u001b[0m colnames \u001b[38;5;241m=\u001b[39m [\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mcolname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m 56\u001b[0m values \u001b[38;5;241m=\u001b[39m [\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mvalue\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n",
66
+ "Cell \u001b[0;32mIn[48], line 24\u001b[0m, in \u001b[0;36minsert_row\u001b[0;34m(conn, tblname, colnames, values)\u001b[0m\n\u001b[1;32m 22\u001b[0m create_table(conn,tblname)\n\u001b[1;32m 23\u001b[0m add_columns(conn,tblname,colnames)\n\u001b[0;32m---> 24\u001b[0m \u001b[43minsert_into\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\u001b[43mtblname\u001b[49m\u001b[43m,\u001b[49m\u001b[43mcolnames\u001b[49m\u001b[43m,\u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m)\u001b[49m\n",
67
+ "Cell \u001b[0;32mIn[48], line 28\u001b[0m, in \u001b[0;36minsert_into\u001b[0;34m(conn, tblname, colnames, values)\u001b[0m\n\u001b[1;32m 26\u001b[0m sql \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mINSERT OR IGNORE INTO \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtblname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m (\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(colnames)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m) VALUES (\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(values)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m)\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 27\u001b[0m \u001b[38;5;28mprint\u001b[39m(sql)\n\u001b[0;32m---> 28\u001b[0m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcursor\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[43msql\u001b[49m\u001b[43m)\u001b[49m\n",
68
+ "\u001b[0;31mOperationalError\u001b[0m: database is locked"
69
+ ]
70
+ }
71
+ ],
72
+ "source": [
73
+ "def read_dict(file):\n",
74
+ " return Json.deserialize(File.read(file))\n",
75
+ "def edit_tblname(tblname):\n",
76
+ " return tblname.replace(\"/\",\"_\").replace(\"[\",\"_\").replace(\"]\",\"_\")\n",
77
+ "def create_table(conn,tblname):\n",
78
+ " sql = f\"CREATE TABLE IF NOT EXISTS '{tblname}' ('ID' INT PRIARY KEY UNIQUE)\"\n",
79
+ " conn.cursor().execute(sql)\n",
80
+ "def add_column(conn,tblname,colname):\n",
81
+ " sql = f\"ALTER TABLE {tblname} ADD COLUMN {colname} TEXT;\"\n",
82
+ " conn.cursor().execute(sql)\n",
83
+ "def add_columns(conn,tblname,colnames):\n",
84
+ " sql = f\"PRAGMA TABLE_INFO({tblname})\"\n",
85
+ " cursor = conn.cursor()\n",
86
+ " cursor.execute(sql)\n",
87
+ " cols = []\n",
88
+ " for row in cursor:\n",
89
+ " cols.append(f\"'{row[1]}'\")\n",
90
+ " for colname in colnames:\n",
91
+ " if not colname in cols:\n",
92
+ " add_column(conn,tblname,colname)\n",
93
+ "def insert_row(conn,tblname,colnames,values):\n",
94
+ " create_table(conn,tblname)\n",
95
+ " add_columns(conn,tblname,colnames)\n",
96
+ " insert_into(conn,tblname,colnames,values)\n",
97
+ "def insert_into(conn,tblname,colnames,values):\n",
98
+ " sql = f\"INSERT OR IGNORE INTO '{tblname}' ({','.join(colnames)}) VALUES ({','.join(values)})\"\n",
99
+ " print(sql)\n",
100
+ " conn.cursor().execute(sql)\n",
101
+ "\n",
102
+ "def main():\n",
103
+ " RESULT_PATH_ROOT = \"/data/Dataset/hkjc/soccer/result/path\"\n",
104
+ " with closing(sqlite3.connect(\"test.db\")) as conn:\n",
105
+ " files = File.get_files(RESULT_PATH_ROOT)\n",
106
+ " progress = Progress(len(files))\n",
107
+ " for file in files:\n",
108
+ " dic = read_dict(file)\n",
109
+ " last_tblname = None\n",
110
+ " for key in dic:\n",
111
+ " value = dic[key]\n",
112
+ " if type(value)==bool:\n",
113
+ " value = str(value)\n",
114
+ " tblname = os.path.dirname(key)\n",
115
+ " tblname = edit_tblname(tblname)\n",
116
+ " colname = os.path.basename(key)\n",
117
+ " colname = edit_tblname(colname)\n",
118
+ " # print(parent,name,value)\n",
119
+ " if last_tblname == None:\n",
120
+ " colnames = [f\"'{colname}'\"]\n",
121
+ " values = [f\"'{value}'\"]\n",
122
+ " elif tblname == last_tblname: \n",
123
+ " colnames.append(f\"'{colname}'\")\n",
124
+ " values.append(f\"'{value}'\")\n",
125
+ " else:\n",
126
+ " insert_row(conn,tblname,colnames,values)\n",
127
+ " colnames = [f\"'{colname}'\"]\n",
128
+ " values = [f\"'{value}'\"]\n",
129
+ " last_tblname = tblname\n",
130
+ " insert_row(conn,tblname,colnames,values)\n",
131
+ " progress.update(1)\n",
132
+ " break\n",
133
+ "\n",
134
+ "main()\n",
135
+ "print(\"done\")"
136
+ ]
137
+ },
138
+ {
139
+ "cell_type": "code",
140
+ "execution_count": 33,
141
+ "id": "d261f797-7fb4-407a-857e-430f534bfa32",
142
+ "metadata": {},
143
+ "outputs": [
144
+ {
145
+ "name": "stdout",
146
+ "output_type": "stream",
147
+ "text": [
148
+ "['T', 'T2']\n",
149
+ "['C']\n",
150
+ "['C', 'C2']\n"
151
+ ]
152
+ }
153
+ ],
154
+ "source": [
155
+ "!rm test.db -r\n",
156
+ "\n",
157
+ "conn = sqlite3.connect(\"test.db\")\n",
158
+ "\n",
159
+ "sql = \"CREATE TABLE IF NOT EXISTS 'T' ('C' INT PRIMARY KEY UNIQUE);\"\n",
160
+ "cursor = conn.cursor()\n",
161
+ "cursor.execute(sql)\n",
162
+ "cursor.close()\n",
163
+ "\n",
164
+ "sql = \"CREATE TABLE IF NOT EXISTS 'T2' ('C2' INT PRIMARY KEY UNIQUE);\"\n",
165
+ "cursor = conn.cursor()\n",
166
+ "cursor.execute(sql)\n",
167
+ "cursor.close()\n",
168
+ "\n",
169
+ "sql = f\"SELECT TBL_NAME FROM SQLITE_MASTER GROUP BY TBL_NAME;\"\n",
170
+ "cursor = conn.cursor()\n",
171
+ "cursor.execute(sql)\n",
172
+ "tblnames = [row[0] for row in cursor]\n",
173
+ "print(tblnames)\n",
174
+ "cursor.close()\n",
175
+ "\n",
176
+ "\n",
177
+ "colnames = []\n",
178
+ "for tblname in tblnames:\n",
179
+ " sql = f\"PRAGMA TABLE_INFO({tblname})\"\n",
180
+ " cursor = conn.cursor()\n",
181
+ " cursor.execute(sql)\n",
182
+ " for row in cursor:\n",
183
+ " colnames.append(row[1])\n",
184
+ " print(colnames)\n",
185
+ " cursor.close()\n",
186
+ " \n",
187
+ "\n",
188
+ "conn.close()"
189
+ ]
190
+ },
191
+ {
192
+ "cell_type": "code",
193
+ "execution_count": null,
194
+ "id": "930632eb-86c7-4d63-b9ae-b1df7d3d43ef",
195
+ "metadata": {},
196
+ "outputs": [],
197
+ "source": []
198
+ },
199
+ {
200
+ "cell_type": "code",
201
+ "execution_count": null,
202
+ "id": "5d8f58f3-5a9f-4cbd-8930-d365a4ec4819",
203
+ "metadata": {},
204
+ "outputs": [],
205
+ "source": []
206
+ }
207
+ ],
208
+ "metadata": {
209
+ "kernelspec": {
210
+ "display_name": "Python 3 (ipykernel)",
211
+ "language": "python",
212
+ "name": "python3"
213
+ },
214
+ "language_info": {
215
+ "codemirror_mode": {
216
+ "name": "ipython",
217
+ "version": 3
218
+ },
219
+ "file_extension": ".py",
220
+ "mimetype": "text/x-python",
221
+ "name": "python",
222
+ "nbconvert_exporter": "python",
223
+ "pygments_lexer": "ipython3",
224
+ "version": "3.9.5"
225
+ }
226
+ },
227
+ "nbformat": 4,
228
+ "nbformat_minor": 5
229
+ }
Project/app/hkjc/soccer/result/import/import-json-2.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Project/app/hkjc/soccer/result/import/import-json.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Project/app/hkjc/soccer/result/import/import.ipynb ADDED
@@ -0,0 +1,229 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": 1,
6
+ "id": "c0a06deb-cb7b-4372-a7fd-675380b30353",
7
+ "metadata": {},
8
+ "outputs": [],
9
+ "source": [
10
+ "%run /data/Project/scripts/common/references.ipynb"
11
+ ]
12
+ },
13
+ {
14
+ "cell_type": "code",
15
+ "execution_count": 48,
16
+ "id": "840f09be-4491-43d6-bca9-7885b4e0befa",
17
+ "metadata": {},
18
+ "outputs": [
19
+ {
20
+ "data": {
21
+ "application/json": {
22
+ "ascii": false,
23
+ "bar_format": null,
24
+ "colour": null,
25
+ "elapsed": 0.004201173782348633,
26
+ "initial": 0,
27
+ "n": 0,
28
+ "ncols": null,
29
+ "nrows": null,
30
+ "postfix": null,
31
+ "prefix": "",
32
+ "rate": null,
33
+ "total": 14325,
34
+ "unit": "it",
35
+ "unit_divisor": 1000,
36
+ "unit_scale": false
37
+ },
38
+ "application/vnd.jupyter.widget-view+json": {
39
+ "model_id": "906cc603a7a24ded9ef7e8f6a3f5c2e7",
40
+ "version_major": 2,
41
+ "version_minor": 0
42
+ },
43
+ "text/plain": [
44
+ " 0%| | 0/14325 [00:00<?, ?it/s]"
45
+ ]
46
+ },
47
+ "metadata": {},
48
+ "output_type": "display_data"
49
+ },
50
+ {
51
+ "name": "stdout",
52
+ "output_type": "stream",
53
+ "text": [
54
+ "INSERT OR IGNORE INTO '_20030802_1__tournaments_1_' ('name','matchescount') VALUES ('SearchMatches','7')\n"
55
+ ]
56
+ },
57
+ {
58
+ "ename": "OperationalError",
59
+ "evalue": "database is locked",
60
+ "output_type": "error",
61
+ "traceback": [
62
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
63
+ "\u001b[0;31mOperationalError\u001b[0m Traceback (most recent call last)",
64
+ "Cell \u001b[0;32mIn[48], line 62\u001b[0m\n\u001b[1;32m 59\u001b[0m progress\u001b[38;5;241m.\u001b[39mupdate(\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m 60\u001b[0m \u001b[38;5;28;01mbreak\u001b[39;00m\n\u001b[0;32m---> 62\u001b[0m \u001b[43mmain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 63\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdone\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
65
+ "Cell \u001b[0;32mIn[48], line 54\u001b[0m, in \u001b[0;36mmain\u001b[0;34m()\u001b[0m\n\u001b[1;32m 52\u001b[0m values\u001b[38;5;241m.\u001b[39mappend(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mvalue\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 53\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m---> 54\u001b[0m \u001b[43minsert_row\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\u001b[43mtblname\u001b[49m\u001b[43m,\u001b[49m\u001b[43mcolnames\u001b[49m\u001b[43m,\u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 55\u001b[0m colnames \u001b[38;5;241m=\u001b[39m [\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mcolname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m 56\u001b[0m values \u001b[38;5;241m=\u001b[39m [\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mvalue\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n",
66
+ "Cell \u001b[0;32mIn[48], line 24\u001b[0m, in \u001b[0;36minsert_row\u001b[0;34m(conn, tblname, colnames, values)\u001b[0m\n\u001b[1;32m 22\u001b[0m create_table(conn,tblname)\n\u001b[1;32m 23\u001b[0m add_columns(conn,tblname,colnames)\n\u001b[0;32m---> 24\u001b[0m \u001b[43minsert_into\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\u001b[43mtblname\u001b[49m\u001b[43m,\u001b[49m\u001b[43mcolnames\u001b[49m\u001b[43m,\u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m)\u001b[49m\n",
67
+ "Cell \u001b[0;32mIn[48], line 28\u001b[0m, in \u001b[0;36minsert_into\u001b[0;34m(conn, tblname, colnames, values)\u001b[0m\n\u001b[1;32m 26\u001b[0m sql \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mINSERT OR IGNORE INTO \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtblname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m (\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(colnames)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m) VALUES (\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(values)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m)\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 27\u001b[0m \u001b[38;5;28mprint\u001b[39m(sql)\n\u001b[0;32m---> 28\u001b[0m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcursor\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[43msql\u001b[49m\u001b[43m)\u001b[49m\n",
68
+ "\u001b[0;31mOperationalError\u001b[0m: database is locked"
69
+ ]
70
+ }
71
+ ],
72
+ "source": [
73
+ "def read_dict(file):\n",
74
+ " return Json.deserialize(File.read(file))\n",
75
+ "def edit_tblname(tblname):\n",
76
+ " return tblname.replace(\"/\",\"_\").replace(\"[\",\"_\").replace(\"]\",\"_\")\n",
77
+ "def create_table(conn,tblname):\n",
78
+ " sql = f\"CREATE TABLE IF NOT EXISTS '{tblname}' ('ID' INT PRIARY KEY UNIQUE)\"\n",
79
+ " conn.cursor().execute(sql)\n",
80
+ "def add_column(conn,tblname,colname):\n",
81
+ " sql = f\"ALTER TABLE {tblname} ADD COLUMN {colname} TEXT;\"\n",
82
+ " conn.cursor().execute(sql)\n",
83
+ "def add_columns(conn,tblname,colnames):\n",
84
+ " sql = f\"PRAGMA TABLE_INFO({tblname})\"\n",
85
+ " cursor = conn.cursor()\n",
86
+ " cursor.execute(sql)\n",
87
+ " cols = []\n",
88
+ " for row in cursor:\n",
89
+ " cols.append(f\"'{row[1]}'\")\n",
90
+ " for colname in colnames:\n",
91
+ " if not colname in cols:\n",
92
+ " add_column(conn,tblname,colname)\n",
93
+ "def insert_row(conn,tblname,colnames,values):\n",
94
+ " create_table(conn,tblname)\n",
95
+ " add_columns(conn,tblname,colnames)\n",
96
+ " insert_into(conn,tblname,colnames,values)\n",
97
+ "def insert_into(conn,tblname,colnames,values):\n",
98
+ " sql = f\"INSERT OR IGNORE INTO '{tblname}' ({','.join(colnames)}) VALUES ({','.join(values)})\"\n",
99
+ " print(sql)\n",
100
+ " conn.cursor().execute(sql)\n",
101
+ "\n",
102
+ "def main():\n",
103
+ " RESULT_PATH_ROOT = \"/data/Dataset/hkjc/soccer/result/path\"\n",
104
+ " with closing(sqlite3.connect(\"test.db\")) as conn:\n",
105
+ " files = File.get_files(RESULT_PATH_ROOT)\n",
106
+ " progress = Progress(len(files))\n",
107
+ " for file in files:\n",
108
+ " dic = read_dict(file)\n",
109
+ " last_tblname = None\n",
110
+ " for key in dic:\n",
111
+ " value = dic[key]\n",
112
+ " if type(value)==bool:\n",
113
+ " value = str(value)\n",
114
+ " tblname = os.path.dirname(key)\n",
115
+ " tblname = edit_tblname(tblname)\n",
116
+ " colname = os.path.basename(key)\n",
117
+ " colname = edit_tblname(colname)\n",
118
+ " # print(parent,name,value)\n",
119
+ " if last_tblname == None:\n",
120
+ " colnames = [f\"'{colname}'\"]\n",
121
+ " values = [f\"'{value}'\"]\n",
122
+ " elif tblname == last_tblname: \n",
123
+ " colnames.append(f\"'{colname}'\")\n",
124
+ " values.append(f\"'{value}'\")\n",
125
+ " else:\n",
126
+ " insert_row(conn,tblname,colnames,values)\n",
127
+ " colnames = [f\"'{colname}'\"]\n",
128
+ " values = [f\"'{value}'\"]\n",
129
+ " last_tblname = tblname\n",
130
+ " insert_row(conn,tblname,colnames,values)\n",
131
+ " progress.update(1)\n",
132
+ " break\n",
133
+ "\n",
134
+ "main()\n",
135
+ "print(\"done\")"
136
+ ]
137
+ },
138
+ {
139
+ "cell_type": "code",
140
+ "execution_count": 33,
141
+ "id": "d261f797-7fb4-407a-857e-430f534bfa32",
142
+ "metadata": {},
143
+ "outputs": [
144
+ {
145
+ "name": "stdout",
146
+ "output_type": "stream",
147
+ "text": [
148
+ "['T', 'T2']\n",
149
+ "['C']\n",
150
+ "['C', 'C2']\n"
151
+ ]
152
+ }
153
+ ],
154
+ "source": [
155
+ "!rm test.db -r\n",
156
+ "\n",
157
+ "conn = sqlite3.connect(\"test.db\")\n",
158
+ "\n",
159
+ "sql = \"CREATE TABLE IF NOT EXISTS 'T' ('C' INT PRIMARY KEY UNIQUE);\"\n",
160
+ "cursor = conn.cursor()\n",
161
+ "cursor.execute(sql)\n",
162
+ "cursor.close()\n",
163
+ "\n",
164
+ "sql = \"CREATE TABLE IF NOT EXISTS 'T2' ('C2' INT PRIMARY KEY UNIQUE);\"\n",
165
+ "cursor = conn.cursor()\n",
166
+ "cursor.execute(sql)\n",
167
+ "cursor.close()\n",
168
+ "\n",
169
+ "sql = f\"SELECT TBL_NAME FROM SQLITE_MASTER GROUP BY TBL_NAME;\"\n",
170
+ "cursor = conn.cursor()\n",
171
+ "cursor.execute(sql)\n",
172
+ "tblnames = [row[0] for row in cursor]\n",
173
+ "print(tblnames)\n",
174
+ "cursor.close()\n",
175
+ "\n",
176
+ "\n",
177
+ "colnames = []\n",
178
+ "for tblname in tblnames:\n",
179
+ " sql = f\"PRAGMA TABLE_INFO({tblname})\"\n",
180
+ " cursor = conn.cursor()\n",
181
+ " cursor.execute(sql)\n",
182
+ " for row in cursor:\n",
183
+ " colnames.append(row[1])\n",
184
+ " print(colnames)\n",
185
+ " cursor.close()\n",
186
+ " \n",
187
+ "\n",
188
+ "conn.close()"
189
+ ]
190
+ },
191
+ {
192
+ "cell_type": "code",
193
+ "execution_count": null,
194
+ "id": "930632eb-86c7-4d63-b9ae-b1df7d3d43ef",
195
+ "metadata": {},
196
+ "outputs": [],
197
+ "source": []
198
+ },
199
+ {
200
+ "cell_type": "code",
201
+ "execution_count": null,
202
+ "id": "5d8f58f3-5a9f-4cbd-8930-d365a4ec4819",
203
+ "metadata": {},
204
+ "outputs": [],
205
+ "source": []
206
+ }
207
+ ],
208
+ "metadata": {
209
+ "kernelspec": {
210
+ "display_name": "Python 3 (ipykernel)",
211
+ "language": "python",
212
+ "name": "python3"
213
+ },
214
+ "language_info": {
215
+ "codemirror_mode": {
216
+ "name": "ipython",
217
+ "version": 3
218
+ },
219
+ "file_extension": ".py",
220
+ "mimetype": "text/x-python",
221
+ "name": "python",
222
+ "nbconvert_exporter": "python",
223
+ "pygments_lexer": "ipython3",
224
+ "version": "3.9.5"
225
+ }
226
+ },
227
+ "nbformat": 4,
228
+ "nbformat_minor": 5
229
+ }
Project/app/hkjc/soccer/result/import/sqlite3helper.ipynb ADDED
The diff for this file is too large to render. See raw diff