mousserlane commited on
Commit
e7f8c6b
·
1 Parent(s): 5ee611e

feat: initial commit

Browse files
.gitignore ADDED
@@ -0,0 +1,165 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ share/python-wheels/
24
+ *.egg-info/
25
+ .installed.cfg
26
+ *.egg
27
+ MANIFEST
28
+ # PyInstaller
29
+ # Usually these files are written by a python script from a template
30
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
31
+ *.manifest
32
+ *.spec
33
+
34
+ # Installer logs
35
+ pip-log.txt
36
+ pip-delete-this-directory.txt
37
+
38
+ # Unit test / coverage reports
39
+ htmlcov/
40
+ .tox/
41
+ .nox/
42
+ .coverage
43
+ .coverage.*
44
+ .cache
45
+ nosetests.xml
46
+ coverage.xml
47
+ *.cover
48
+ *.py,cover
49
+ .hypothesis/
50
+ .pytest_cache/
51
+ cover/
52
+
53
+ # Translations
54
+ *.mo
55
+ *.pot
56
+
57
+ # Django stuff:
58
+ *.log
59
+ local_settings.py
60
+ db.sqlite3
61
+ db.sqlite3-journal
62
+
63
+ # Flask stuff:
64
+ instance/
65
+ .webassets-cache
66
+
67
+ # Scrapy stuff:
68
+ .scrapy
69
+
70
+ # Sphinx documentation
71
+ docs/_build/
72
+
73
+ # PyBuilder
74
+ .pybuilder/
75
+ target/
76
+
77
+ # Jupyter Notebook
78
+ .ipynb_checkpoints
79
+
80
+ # IPython
81
+ profile_default/
82
+ ipython_config.py
83
+
84
+ sparrow-data/docs/input/sroie
85
+
86
+ # pyenv
87
+ # For a library or package, you might want to ignore these files since the code is
88
+ # intended to run in multiple environments; otherwise, check them in:
89
+ # .python-version
90
+
91
+ # pipenv
92
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
93
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
94
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
95
+ # install all needed dependencies.
96
+ #Pipfile.lock
97
+
98
+ # poetry
99
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
100
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
101
+ # commonly ignored for libraries.
102
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
103
+ #poetry.lock
104
+
105
+ # pdm
106
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
107
+ #pdm.lock
108
+ # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
109
+ # in version control.
110
+ # https://pdm.fming.dev/#use-with-ide
111
+ .pdm.toml
112
+
113
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
114
+ __pypackages__/
115
+
116
+ # Celery stuff
117
+ celerybeat-schedule
118
+ celerybeat.pid
119
+
120
+ # SageMath parsed files
121
+ *.sage.py
122
+
123
+ # Environments
124
+ .env
125
+ .venv
126
+ .sb-sparrow-donut
127
+ env/
128
+ venv/
129
+ ENV/
130
+ env.bak/
131
+ venv.bak/
132
+
133
+ # Spyder project settings
134
+ .spyderproject
135
+ .spyproject
136
+
137
+ # Rope project settings
138
+ .ropeproject
139
+
140
+ # mkdocs documentation
141
+ /site
142
+
143
+ # mypy
144
+ .mypy_cache/
145
+ .dmypy.json
146
+ dmypy.json
147
+
148
+ # Pyre type checker
149
+ .pyre/
150
+
151
+ # pytype static type analyzer
152
+ .pytype/
153
+
154
+ # Cython debug symbols
155
+ cython_debug/
156
+
157
+ # PyCharm
158
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
159
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
160
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
161
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
162
+ #.idea/
163
+ .DS_Store
164
+
165
+ sparrow-ui/.env_sparrow_ui/
.idea/.gitignore ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ # Default ignored files
2
+ /shelf/
3
+ /workspace.xml
4
+ # Editor-based HTTP Client requests
5
+ /httpRequests/
6
+ # Datasource local storage ignored files
7
+ /dataSources/
8
+ /dataSources.local.xml
.streamlit/config.toml ADDED
@@ -0,0 +1,183 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Below are all the sections and options you can have in ~/.streamlit/config.toml.
2
+
3
+ [global]
4
+
5
+ # By default, Streamlit checks if the Python watchdog module is available and, if not, prints a warning asking for you to install it. The watchdog module is not required, but highly recommended. It improves Streamlit's ability to detect changes to files in your filesystem.
6
+ # If you'd like to turn off this warning, set this to True.
7
+ # Default: false
8
+ disableWatchdogWarning = true
9
+
10
+ # If True, will show a warning when you run a Streamlit-enabled script via "python my_script.py".
11
+ # Default: true
12
+ # showWarningOnDirectExecution = true
13
+
14
+ # DataFrame serialization.
15
+ # Acceptable values: - 'legacy': Serialize DataFrames using Streamlit's custom format. Slow but battle-tested. - 'arrow': Serialize DataFrames using Apache Arrow. Much faster and versatile.
16
+ # Default: "arrow"
17
+ # dataFrameSerialization = "arrow"
18
+
19
+
20
+ [logger]
21
+
22
+ # Level of logging: 'error', 'warning', 'info', or 'debug'.
23
+ # Default: 'info'
24
+ # level = "info"
25
+
26
+ # String format for logging messages. If logger.datetimeFormat is set, logger messages will default to `%(asctime)s.%(msecs)03d %(message)s`. See [Python's documentation](https://docs.python.org/2.6/library/logging.html#formatter-objects) for available attributes.
27
+ # Default: "%(asctime)s %(message)s"
28
+ # messageFormat = "%(asctime)s %(message)s"
29
+
30
+
31
+ [client]
32
+
33
+ # Whether to enable st.cache.
34
+ # Default: true
35
+ # caching = true
36
+
37
+ # If false, makes your Streamlit script not draw to a Streamlit app.
38
+ # Default: true
39
+ # displayEnabled = true
40
+
41
+ # Controls whether uncaught app exceptions are displayed in the browser. By default, this is set to True and Streamlit displays app exceptions and associated tracebacks in the browser.
42
+ # If set to False, an exception will result in a generic message being shown in the browser, and exceptions and tracebacks will be printed to the console only.
43
+ # Default: true
44
+ # showErrorDetails = true
45
+
46
+
47
+ [runner]
48
+
49
+ # Allows you to type a variable or string by itself in a single line of Python code to write it to the app.
50
+ # Default: true
51
+ # magicEnabled = true
52
+
53
+ # Install a Python tracer to allow you to stop or pause your script at any point and introspect it. As a side-effect, this slows down your script's execution.
54
+ # Default: false
55
+ # installTracer = false
56
+
57
+ # Sets the MPLBACKEND environment variable to Agg inside Streamlit to prevent Python crashing.
58
+ # Default: true
59
+ # fixMatplotlib = true
60
+
61
+ # Run the Python Garbage Collector after each script execution. This can help avoid excess memory use in Streamlit apps, but could introduce delay in rerunning the app script for high-memory-use applications.
62
+ # Default: true
63
+ # postScriptGC = true
64
+
65
+ # Handle script rerun requests immediately, rather than waiting for script execution to reach a yield point. Enabling this will make Streamlit much more responsive to user interaction, but it can lead to race conditions in apps that mutate session_state data outside of explicit session_state assignment statements.
66
+ # Default: false
67
+ fastReruns = true
68
+
69
+
70
+ [server]
71
+
72
+ # List of folders that should not be watched for changes. This impacts both "Run on Save" and @st.cache.
73
+ # Relative paths will be taken as relative to the current working directory.
74
+ # Example: ['/home/user1/env', 'relative/path/to/folder']
75
+ # Default: []
76
+ # folderWatchBlacklist = []
77
+
78
+ # Change the type of file watcher used by Streamlit, or turn it off completely.
79
+ # Allowed values: * "auto" : Streamlit will attempt to use the watchdog module, and falls back to polling if watchdog is not available. * "watchdog" : Force Streamlit to use the watchdog module. * "poll" : Force Streamlit to always use polling. * "none" : Streamlit will not watch files.
80
+ # Default: "auto"
81
+ fileWatcherType = "auto"
82
+
83
+ # Symmetric key used to produce signed cookies. If deploying on multiple replicas, this should be set to the same value across all replicas to ensure they all share the same secret.
84
+ # Default: randomly generated secret key.
85
+ # cookieSecret = "80f9eb91f1eb64e26f0e46148556bf493ccde5fe27712bbcbebf57948852b5fe"
86
+
87
+ # If false, will attempt to open a browser window on start.
88
+ # Default: false unless (1) we are on a Linux box where DISPLAY is unset, or (2) we are running in the Streamlit Atom plugin.
89
+ # headless = false
90
+
91
+ # Automatically rerun script when the file is modified on disk.
92
+ # Default: false
93
+ # runOnSave = false
94
+
95
+ # The address where the server will listen for client and browser connections. Use this if you want to bind the server to a specific address. If set, the server will only be accessible from this address, and not from any aliases (like localhost).
96
+ # Default: (unset)
97
+ # address =
98
+
99
+ # The port where the server will listen for browser connections.
100
+ # Default: 8501
101
+ port = 7860
102
+
103
+ # The base path for the URL where Streamlit should be served from.
104
+ # Default: ""
105
+ # baseUrlPath = ""
106
+
107
+ # Enables support for Cross-Origin Request Sharing (CORS) protection, for added security.
108
+ # Due to conflicts between CORS and XSRF, if `server.enableXsrfProtection` is on and `server.enableCORS` is off at the same time, we will prioritize `server.enableXsrfProtection`.
109
+ # Default: true
110
+ # enableCORS = true
111
+
112
+ # Enables support for Cross-Site Request Forgery (XSRF) protection, for added security.
113
+ # Due to conflicts between CORS and XSRF, if `server.enableXsrfProtection` is on and `server.enableCORS` is off at the same time, we will prioritize `server.enableXsrfProtection`.
114
+ # Default: true
115
+ # enableXsrfProtection = true
116
+
117
+ # Max size, in megabytes, for files uploaded with the file_uploader.
118
+ # Default: 200
119
+ maxUploadSize = 10
120
+
121
+ # Max size, in megabytes, of messages that can be sent via the WebSocket connection.
122
+ # Default: 200
123
+ # maxMessageSize = 200
124
+
125
+ # Enables support for websocket compression.
126
+ # Default: false
127
+ # enableWebsocketCompression = false
128
+
129
+
130
+ [browser]
131
+
132
+ # Internet address where users should point their browsers in order to connect to the app. Can be IP address or DNS name and path.
133
+ # This is used to: - Set the correct URL for CORS and XSRF protection purposes. - Show the URL on the terminal - Open the browser
134
+ # Default: "localhost"
135
+ # serverAddress = "localhost"
136
+
137
+ # Whether to send usage statistics to Streamlit.
138
+ # Default: true
139
+ gatherUsageStats = false
140
+
141
+ # Port where users should point their browsers in order to connect to the app.
142
+ # This is used to: - Set the correct URL for CORS and XSRF protection purposes. - Show the URL on the terminal - Open the browser
143
+ # Default: whatever value is set in server.port.
144
+ # serverPort = 8501
145
+
146
+
147
+ [mapbox]
148
+
149
+ # Configure Streamlit to use a custom Mapbox token for elements like st.pydeck_chart and st.map. To get a token for yourself, create an account at https://mapbox.com. It's free (for moderate usage levels)!
150
+ # Default: ""
151
+ # token = ""
152
+
153
+
154
+ [deprecation]
155
+
156
+ # Set to false to disable the deprecation warning for the file uploader encoding.
157
+ # Default: true
158
+ # showfileUploaderEncoding = true
159
+
160
+ # Set to false to disable the deprecation warning for using the global pyplot instance.
161
+ # Default: true
162
+ # showPyplotGlobalUse = true
163
+
164
+
165
+ [theme]
166
+
167
+ # The preset Streamlit theme that your custom theme inherits from. One of "light" or "dark".
168
+ # base =
169
+
170
+ # Primary accent color for interactive elements.
171
+ # primaryColor = "#F633"
172
+
173
+ # Background color for the main content area.
174
+ backgroundColor = "#FFFFFF"
175
+
176
+ # Background color used for the sidebar and most interactive widgets.
177
+ secondaryBackgroundColor = "#F0F2F6"
178
+
179
+ # Color used for almost all text.
180
+ textColor = "#262730"
181
+
182
+ # Font family for all text in the app, except code blocks. One of "sans serif", "serif", or "monospace".
183
+ font = "sans serif"
config.py ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+
3
+
4
+ class Settings:
5
+ # sparrow_key: str = os.environ()
6
+ api_url: str = os.environ.get("API_URL")
7
+ huggingface_key: str = os.environ.get("HF_TOKEN")
8
+
9
+
10
+ settings = Settings()
config/config.toml ADDED
@@ -0,0 +1,183 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Below are all the sections and options you can have in ~/.streamlit/config.toml.
2
+
3
+ [global]
4
+
5
+ # By default, Streamlit checks if the Python watchdog module is available and, if not, prints a warning asking for you to install it. The watchdog module is not required, but highly recommended. It improves Streamlit's ability to detect changes to files in your filesystem.
6
+ # If you'd like to turn off this warning, set this to True.
7
+ # Default: false
8
+ disableWatchdogWarning = true
9
+
10
+ # If True, will show a warning when you run a Streamlit-enabled script via "python my_script.py".
11
+ # Default: true
12
+ # showWarningOnDirectExecution = true
13
+
14
+ # DataFrame serialization.
15
+ # Acceptable values: - 'legacy': Serialize DataFrames using Streamlit's custom format. Slow but battle-tested. - 'arrow': Serialize DataFrames using Apache Arrow. Much faster and versatile.
16
+ # Default: "arrow"
17
+ # dataFrameSerialization = "arrow"
18
+
19
+
20
+ [logger]
21
+
22
+ # Level of logging: 'error', 'warning', 'info', or 'debug'.
23
+ # Default: 'info'
24
+ # level = "info"
25
+
26
+ # String format for logging messages. If logger.datetimeFormat is set, logger messages will default to `%(asctime)s.%(msecs)03d %(message)s`. See [Python's documentation](https://docs.python.org/2.6/library/logging.html#formatter-objects) for available attributes.
27
+ # Default: "%(asctime)s %(message)s"
28
+ # messageFormat = "%(asctime)s %(message)s"
29
+
30
+
31
+ [client]
32
+
33
+ # Whether to enable st.cache.
34
+ # Default: true
35
+ # caching = true
36
+
37
+ # If false, makes your Streamlit script not draw to a Streamlit app.
38
+ # Default: true
39
+ # displayEnabled = true
40
+
41
+ # Controls whether uncaught app exceptions are displayed in the browser. By default, this is set to True and Streamlit displays app exceptions and associated tracebacks in the browser.
42
+ # If set to False, an exception will result in a generic message being shown in the browser, and exceptions and tracebacks will be printed to the console only.
43
+ # Default: true
44
+ # showErrorDetails = true
45
+
46
+
47
+ [runner]
48
+
49
+ # Allows you to type a variable or string by itself in a single line of Python code to write it to the app.
50
+ # Default: true
51
+ # magicEnabled = true
52
+
53
+ # Install a Python tracer to allow you to stop or pause your script at any point and introspect it. As a side-effect, this slows down your script's execution.
54
+ # Default: false
55
+ # installTracer = false
56
+
57
+ # Sets the MPLBACKEND environment variable to Agg inside Streamlit to prevent Python crashing.
58
+ # Default: true
59
+ # fixMatplotlib = true
60
+
61
+ # Run the Python Garbage Collector after each script execution. This can help avoid excess memory use in Streamlit apps, but could introduce delay in rerunning the app script for high-memory-use applications.
62
+ # Default: true
63
+ # postScriptGC = true
64
+
65
+ # Handle script rerun requests immediately, rather than waiting for script execution to reach a yield point. Enabling this will make Streamlit much more responsive to user interaction, but it can lead to race conditions in apps that mutate session_state data outside of explicit session_state assignment statements.
66
+ # Default: false
67
+ fastReruns = true
68
+
69
+
70
+ [server]
71
+
72
+ # List of folders that should not be watched for changes. This impacts both "Run on Save" and @st.cache.
73
+ # Relative paths will be taken as relative to the current working directory.
74
+ # Example: ['/home/user1/env', 'relative/path/to/folder']
75
+ # Default: []
76
+ # folderWatchBlacklist = []
77
+
78
+ # Change the type of file watcher used by Streamlit, or turn it off completely.
79
+ # Allowed values: * "auto" : Streamlit will attempt to use the watchdog module, and falls back to polling if watchdog is not available. * "watchdog" : Force Streamlit to use the watchdog module. * "poll" : Force Streamlit to always use polling. * "none" : Streamlit will not watch files.
80
+ # Default: "auto"
81
+ fileWatcherType = "auto"
82
+
83
+ # Symmetric key used to produce signed cookies. If deploying on multiple replicas, this should be set to the same value across all replicas to ensure they all share the same secret.
84
+ # Default: randomly generated secret key.
85
+ # cookieSecret = "80f9eb91f1eb64e26f0e46148556bf493ccde5fe27712bbcbebf57948852b5fe"
86
+
87
+ # If false, will attempt to open a browser window on start.
88
+ # Default: false unless (1) we are on a Linux box where DISPLAY is unset, or (2) we are running in the Streamlit Atom plugin.
89
+ # headless = false
90
+
91
+ # Automatically rerun script when the file is modified on disk.
92
+ # Default: false
93
+ # runOnSave = false
94
+
95
+ # The address where the server will listen for client and browser connections. Use this if you want to bind the server to a specific address. If set, the server will only be accessible from this address, and not from any aliases (like localhost).
96
+ # Default: (unset)
97
+ # address =
98
+
99
+ # The port where the server will listen for browser connections.
100
+ # Default: 8501
101
+ port = 7860
102
+
103
+ # The base path for the URL where Streamlit should be served from.
104
+ # Default: ""
105
+ # baseUrlPath = ""
106
+
107
+ # Enables support for Cross-Origin Request Sharing (CORS) protection, for added security.
108
+ # Due to conflicts between CORS and XSRF, if `server.enableXsrfProtection` is on and `server.enableCORS` is off at the same time, we will prioritize `server.enableXsrfProtection`.
109
+ # Default: true
110
+ # enableCORS = true
111
+
112
+ # Enables support for Cross-Site Request Forgery (XSRF) protection, for added security.
113
+ # Due to conflicts between CORS and XSRF, if `server.enableXsrfProtection` is on and `server.enableCORS` is off at the same time, we will prioritize `server.enableXsrfProtection`.
114
+ # Default: true
115
+ # enableXsrfProtection = true
116
+
117
+ # Max size, in megabytes, for files uploaded with the file_uploader.
118
+ # Default: 200
119
+ maxUploadSize = 10
120
+
121
+ # Max size, in megabytes, of messages that can be sent via the WebSocket connection.
122
+ # Default: 200
123
+ # maxMessageSize = 200
124
+
125
+ # Enables support for websocket compression.
126
+ # Default: false
127
+ # enableWebsocketCompression = false
128
+
129
+
130
+ [browser]
131
+
132
+ # Internet address where users should point their browsers in order to connect to the app. Can be IP address or DNS name and path.
133
+ # This is used to: - Set the correct URL for CORS and XSRF protection purposes. - Show the URL on the terminal - Open the browser
134
+ # Default: "localhost"
135
+ # serverAddress = "localhost"
136
+
137
+ # Whether to send usage statistics to Streamlit.
138
+ # Default: true
139
+ gatherUsageStats = false
140
+
141
+ # Port where users should point their browsers in order to connect to the app.
142
+ # This is used to: - Set the correct URL for CORS and XSRF protection purposes. - Show the URL on the terminal - Open the browser
143
+ # Default: whatever value is set in server.port.
144
+ # serverPort = 8501
145
+
146
+
147
+ [mapbox]
148
+
149
+ # Configure Streamlit to use a custom Mapbox token for elements like st.pydeck_chart and st.map. To get a token for yourself, create an account at https://mapbox.com. It's free (for moderate usage levels)!
150
+ # Default: ""
151
+ # token = ""
152
+
153
+
154
+ [deprecation]
155
+
156
+ # Set to false to disable the deprecation warning for the file uploader encoding.
157
+ # Default: true
158
+ # showfileUploaderEncoding = true
159
+
160
+ # Set to false to disable the deprecation warning for using the global pyplot instance.
161
+ # Default: true
162
+ # showPyplotGlobalUse = true
163
+
164
+
165
+ [theme]
166
+
167
+ # The preset Streamlit theme that your custom theme inherits from. One of "light" or "dark".
168
+ # base =
169
+
170
+ # Primary accent color for interactive elements.
171
+ # primaryColor = "#F633"
172
+
173
+ # Background color for the main content area.
174
+ backgroundColor = "#FFFFFF"
175
+
176
+ # Background color used for the sidebar and most interactive widgets.
177
+ secondaryBackgroundColor = "#F0F2F6"
178
+
179
+ # Color used for almost all text.
180
+ textColor = "#262730"
181
+
182
+ # Font family for all text in the app, except code blocks. One of "sans serif", "serif", or "monospace".
183
+ font = "sans serif"
docs/groups.json ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "groups": [
3
+ {
4
+ "id": 19,
5
+ "name": "items_row11",
6
+ "description": ""
7
+ },
8
+ {
9
+ "id": 18,
10
+ "name": "sub_items_row3_items_row2",
11
+ "description": "Sub items in row 3 in items row 2"
12
+ },
13
+ {
14
+ "id": 17,
15
+ "name": "sub_items_row2_items_row2",
16
+ "description": "Sub items in row 2 in items row 2"
17
+ },
18
+ {
19
+ "id": 16,
20
+ "name": "sub_items_row1_items_row2",
21
+ "description": "Sub items in row 1 in items row 2"
22
+ },
23
+ {
24
+ "id": 15,
25
+ "name": "sub_items_row3_items_row1",
26
+ "description": "Sub items in row 3 in items row 1"
27
+ },
28
+ {
29
+ "id": 14,
30
+ "name": "sub_items_row2_items_row1",
31
+ "description": "Sub items in row 2 in items row 1"
32
+ },
33
+ {
34
+ "id": 13,
35
+ "name": "sub_items_row1_items_row1",
36
+ "description": "Sub items in row 1 in items row 1"
37
+ },
38
+ {
39
+ "id": 12,
40
+ "name": "header",
41
+ "description": "Header"
42
+ },
43
+ {
44
+ "id": 1,
45
+ "name": "items_row1",
46
+ "description": "Items in row 1"
47
+ },
48
+ {
49
+ "id": 2,
50
+ "name": "items_row2",
51
+ "description": "Items in row 2"
52
+ },
53
+ {
54
+ "id": 3,
55
+ "name": "items_row3",
56
+ "description": "Items in row 3"
57
+ },
58
+ {
59
+ "id": 4,
60
+ "name": "items_row4",
61
+ "description": "Items in row 4"
62
+ },
63
+ {
64
+ "id": 5,
65
+ "name": "items_row5",
66
+ "description": "Items in row 5"
67
+ },
68
+ {
69
+ "id": 6,
70
+ "name": "items_row6",
71
+ "description": "Items in row 6"
72
+ },
73
+ {
74
+ "id": 7,
75
+ "name": "items_row7",
76
+ "description": "Items in row 7"
77
+ },
78
+ {
79
+ "id": 8,
80
+ "name": "items_row8",
81
+ "description": "Items in row 8"
82
+ },
83
+ {
84
+ "id": 9,
85
+ "name": "items_row9",
86
+ "description": "Items in row 9"
87
+ },
88
+ {
89
+ "id": 10,
90
+ "name": "items_row10",
91
+ "description": "Items in row 10"
92
+ },
93
+ {
94
+ "id": 11,
95
+ "name": "summary",
96
+ "description": "Summary"
97
+ }
98
+ ]
99
+ }
docs/json/key/receipt_1.json ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "header": {
3
+ "restaurant_name": "Yakiniku Like Grand Indonesia",
4
+ "date": "27-01-2024 12:21"
5
+ },
6
+ "items": [
7
+ {
8
+ "item_qty": "1",
9
+ "item_name": "COUPLE SET 300GR",
10
+ "item_price": "269.000",
11
+ "subitems": [
12
+ {
13
+ "item_qty": "1",
14
+ "item_name": "OCHA COLD (Free Refill)",
15
+ "item_price": "15.000"
16
+ }
17
+ ]
18
+ },
19
+ {
20
+ "item_qty": "1",
21
+ "item_name": "OCHA HOT (Free Refill)",
22
+ "item_price": "15.000"
23
+ }
24
+ ],
25
+ "subtotal": "299.000",
26
+ "service": "8.970",
27
+ "tax": "30.797",
28
+ "total": "338.767"
29
+ }
docs/json/receipt_1.json ADDED
@@ -0,0 +1,203 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "meta": {
3
+ "version": "v1.0",
4
+ "split": "train",
5
+ "image_id": 21,
6
+ "image_size": {
7
+ "width": 720,
8
+ "height": 1600
9
+ }
10
+ },
11
+ "words": [
12
+ {
13
+ "rect": {
14
+ "x1": 143,
15
+ "y1": 536,
16
+ "x2": 557,
17
+ "y2": 577
18
+ },
19
+ "value": "Yakiniku Like Grand Indonesia",
20
+ "label": "header:restaurant_name"
21
+ },
22
+ {
23
+ "rect": {
24
+ "x1": 210,
25
+ "y1": 595,
26
+ "x2": 444,
27
+ "y2": 638
28
+ },
29
+ "value": "27-01-2024 12:21",
30
+ "label": "header:date"
31
+ },
32
+ {
33
+ "rect": {
34
+ "x1": 30,
35
+ "y1": 859,
36
+ "x2": 64,
37
+ "y2": 892
38
+ },
39
+ "value": "1",
40
+ "label": "items_row1:item_qty"
41
+ },
42
+ {
43
+ "rect": {
44
+ "x1": 109,
45
+ "y1": 855,
46
+ "x2": 350,
47
+ "y2": 894
48
+ },
49
+ "value": "COUPLE SET 300GR",
50
+ "label": "items_row1:item_name"
51
+ },
52
+ {
53
+ "rect": {
54
+ "x1": 585,
55
+ "y1": 846,
56
+ "x2": 702,
57
+ "y2": 890
58
+ },
59
+ "value": "269.000",
60
+ "label": "items_row1:item_price"
61
+ },
62
+ {
63
+ "rect": {
64
+ "x1": 108,
65
+ "y1": 895,
66
+ "x2": 139,
67
+ "y2": 932
68
+ },
69
+ "value": "1",
70
+ "label": "items_row1:subitem_row1:item_qty"
71
+ },
72
+ {
73
+ "rect": {
74
+ "x1": 160,
75
+ "y1": 893,
76
+ "x2": 498,
77
+ "y2": 928
78
+ },
79
+ "value": "OCHA COLD (Free Refill)",
80
+ "label": "items_row1:subitem_row1:item_name"
81
+ },
82
+ {
83
+ "rect": {
84
+ "x1": 588,
85
+ "y1": 893,
86
+ "x2": 704,
87
+ "y2": 923
88
+ },
89
+ "value": "15.000",
90
+ "label": "items_row1:subitem_row1:item_price"
91
+ },
92
+ {
93
+ "rect": {
94
+ "x1": 33,
95
+ "y1": 924,
96
+ "x2": 59,
97
+ "y2": 963
98
+ },
99
+ "value": "1",
100
+ "label": "items_row2:item_qty"
101
+ },
102
+ {
103
+ "rect": {
104
+ "x1": 107,
105
+ "y1": 928,
106
+ "x2": 437,
107
+ "y2": 961
108
+ },
109
+ "value": "OCHA HOT (Free Refill)",
110
+ "label": "items_row2:item_name"
111
+ },
112
+ {
113
+ "rect": {
114
+ "x1": 591,
115
+ "y1": 926,
116
+ "x2": 707,
117
+ "y2": 958
118
+ },
119
+ "value": "15.000",
120
+ "label": "items_row2:item_price"
121
+ },
122
+ {
123
+ "rect": {
124
+ "x1": 360,
125
+ "y1": 1021,
126
+ "x2": 494,
127
+ "y2": 1055
128
+ },
129
+ "value": "Subtotal",
130
+ "label": ""
131
+ },
132
+ {
133
+ "rect": {
134
+ "x1": 582,
135
+ "y1": 1023,
136
+ "x2": 700,
137
+ "y2": 1056
138
+ },
139
+ "value": "299.000",
140
+ "label": "subtotal"
141
+ },
142
+ {
143
+ "rect": {
144
+ "x1": 439,
145
+ "y1": 1060,
146
+ "x2": 501,
147
+ "y2": 1094
148
+ },
149
+ "value": "SC",
150
+ "label": ""
151
+ },
152
+ {
153
+ "rect": {
154
+ "x1": 594,
155
+ "y1": 1061,
156
+ "x2": 703,
157
+ "y2": 1085
158
+ },
159
+ "value": "8.970",
160
+ "label": "service"
161
+ },
162
+ {
163
+ "rect": {
164
+ "x1": 434,
165
+ "y1": 1098,
166
+ "x2": 494,
167
+ "y2": 1124
168
+ },
169
+ "value": "PB1",
170
+ "label": ""
171
+ },
172
+ {
173
+ "rect": {
174
+ "x1": 595,
175
+ "y1": 1092,
176
+ "x2": 701,
177
+ "y2": 1121
178
+ },
179
+ "value": "30.797",
180
+ "label": "tax"
181
+ },
182
+ {
183
+ "rect": {
184
+ "x1": 323,
185
+ "y1": 1151,
186
+ "x2": 494,
187
+ "y2": 1219
188
+ },
189
+ "value": "Grand Total",
190
+ "label": ""
191
+ },
192
+ {
193
+ "rect": {
194
+ "x1": 584,
195
+ "y1": 1150,
196
+ "x2": 688,
197
+ "y2": 1194
198
+ },
199
+ "value": "338.767",
200
+ "label": "total"
201
+ }
202
+ ]
203
+ }
docs/labels.json ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "labels": [
3
+ {
4
+ "id": 21,
5
+ "name": "item_subtotal",
6
+ "description": ""
7
+ },
8
+ {
9
+ "id": 20,
10
+ "name": "service",
11
+ "description": ""
12
+ },
13
+ {
14
+ "id": 19,
15
+ "name": "tax",
16
+ "description": ""
17
+ },
18
+ {
19
+ "id": 18,
20
+ "name": "restaurant_name",
21
+ "description": ""
22
+ },
23
+ {
24
+ "id": 17,
25
+ "name": "item_name",
26
+ "description": ""
27
+ },
28
+ {
29
+ "id": 2,
30
+ "name": "date",
31
+ "description": "Invoice Date"
32
+ },
33
+ {
34
+ "id": 8,
35
+ "name": "item_desc",
36
+ "description": "Item Description"
37
+ },
38
+ {
39
+ "id": 9,
40
+ "name": "item_qty",
41
+ "description": "Item Quantity"
42
+ },
43
+ {
44
+ "id": 10,
45
+ "name": "item_value",
46
+ "description": "Item Total Price"
47
+ },
48
+ {
49
+ "id": 14,
50
+ "name": "total",
51
+ "description": "Total Net Worth"
52
+ },
53
+ {
54
+ "id": 16,
55
+ "name": "subtotal",
56
+ "description": "Total Gross Worth"
57
+ }
58
+ ]
59
+ }
docs/status.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "annotations": [
3
+ {
4
+ "completed": 501,
5
+ "in_progress": 499,
6
+ "total": 1000
7
+ }
8
+ ]
9
+ }
docs/subgroups.json ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "subgroups": [
3
+ {
4
+ "id": 3,
5
+ "name": "subitem_row3",
6
+ "description": ""
7
+ },
8
+ {
9
+ "id": 2,
10
+ "name": "subitem_row2",
11
+ "description": ""
12
+ },
13
+ {
14
+ "id": 1,
15
+ "name": "subitem_row1",
16
+ "description": "Sub Item Row 1"
17
+ }
18
+ ]
19
+ }
docs/visitors.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"meta": {"visitors": 4102}}
main.py ADDED
@@ -0,0 +1,233 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+ from streamlit_option_menu import option_menu
3
+ from tools.utilities import load_css
4
+ import json
5
+
6
+ from views.dashboard import Dashboard
7
+ from views.data_annotation import DataAnnotation
8
+ from views.model_training import ModelTraining
9
+ from views.model_tuning import ModelTuning
10
+ from views.data_inference import DataInference
11
+ from views.setup import Setup
12
+ from views.data_review import DataReview
13
+ from views.about import About
14
+
15
+ import streamlit_javascript as st_js
16
+
17
+ st.set_page_config(page_title="Sparrow", page_icon="favicon.ico", layout="wide")
18
+
19
+ load_css()
20
+
21
+
22
+ class Model:
23
+ menuTitle = "Sparrow"
24
+ # option1 = "Dashboard"
25
+ option2 = "Data Annotation"
26
+ option3 = "Model Training"
27
+ option4 = "Model Tuning"
28
+ # option5 = "Inference"
29
+ # option6 = "Data Review"
30
+ option7 = "Setup"
31
+ # option8 = "About"
32
+
33
+ menuIcon = "menu-up"
34
+ icon1 = "speedometer"
35
+ icon2 = "activity"
36
+ icon3 = "motherboard"
37
+ icon4 = "graph-up-arrow"
38
+ icon5 = "journal-arrow-down"
39
+ icon6 = "droplet"
40
+ icon7 = "clipboard-data"
41
+ icon8 = "chat"
42
+
43
+
44
+ def view(model):
45
+ with st.sidebar:
46
+ menuItem = option_menu(
47
+ model.menuTitle,
48
+ [
49
+ # model.option1,
50
+ model.option2,
51
+ # model.option5,
52
+ # model.option6,
53
+ model.option7,
54
+ # model.option8,
55
+ ],
56
+ icons=[
57
+ # model.icon1,
58
+ model.icon2,
59
+ # model.icon5,
60
+ # model.icon6,
61
+ model.icon7,
62
+ # model.icon8,
63
+ ],
64
+ menu_icon=model.menuIcon,
65
+ default_index=0,
66
+ styles={
67
+ "container": {"padding": "5!important", "background-color": "#fafafa"},
68
+ "icon": {"color": "black", "font-size": "25px"},
69
+ "nav-link": {
70
+ "font-size": "16px",
71
+ "text-align": "left",
72
+ "margin": "0px",
73
+ "--hover-color": "#eee",
74
+ },
75
+ "nav-link-selected": {"background-color": "#037ffc"},
76
+ },
77
+ )
78
+
79
+ # if menuItem == model.option1:
80
+ # Dashboard().view(Dashboard.Model())
81
+ # logout_widget()
82
+
83
+ if menuItem == model.option2:
84
+ if (
85
+ "ui_width" not in st.session_state
86
+ or "device_type" not in st.session_state
87
+ or "device_width" not in st.session_state
88
+ ):
89
+ # Get UI width
90
+ ui_width = st_js.st_javascript("window.innerWidth", key="ui_width_comp")
91
+ device_width = st_js.st_javascript(
92
+ "window.screen.width", key="device_width_comp"
93
+ )
94
+
95
+ if ui_width > 0 and device_width > 0:
96
+ # Add 20% of current screen width to compensate for the sidebar
97
+ ui_width = round(ui_width + (20 * ui_width / 100))
98
+
99
+ if device_width > 768:
100
+ device_type = "desktop"
101
+ else:
102
+ device_type = "mobile"
103
+
104
+ st.session_state["ui_width"] = ui_width
105
+ st.session_state["device_type"] = device_type
106
+ st.session_state["device_width"] = device_width
107
+
108
+ st.experimental_rerun()
109
+ else:
110
+ DataAnnotation().view(
111
+ DataAnnotation.Model(),
112
+ st.session_state["ui_width"],
113
+ st.session_state["device_type"],
114
+ st.session_state["device_width"],
115
+ )
116
+ logout_widget()
117
+
118
+ if menuItem == model.option3:
119
+ ModelTraining().view(ModelTraining.Model())
120
+ logout_widget()
121
+
122
+ if menuItem == model.option4:
123
+ ModelTuning().view(ModelTuning.Model())
124
+ logout_widget()
125
+
126
+ # if menuItem == model.option5:
127
+ # if (
128
+ # "ui_width" not in st.session_state
129
+ # or "device_type" not in st.session_state
130
+ # or "device_width" not in st.session_state
131
+ # ):
132
+ # # Get UI width
133
+ # ui_width = st_js.st_javascript("window.innerWidth", key="ui_width_comp")
134
+ # device_width = st_js.st_javascript(
135
+ # "window.screen.width", key="device_width_comp"
136
+ # )
137
+
138
+ # if ui_width > 0 and device_width > 0:
139
+ # # Add 20% of current screen width to compensate for the sidebar
140
+ # ui_width = round(ui_width + (20 * ui_width / 100))
141
+
142
+ # if device_width > 768:
143
+ # device_type = "desktop"
144
+ # else:
145
+ # device_type = "mobile"
146
+
147
+ # st.session_state["ui_width"] = ui_width
148
+ # st.session_state["device_type"] = device_type
149
+ # st.session_state["device_width"] = device_width
150
+
151
+ # st.experimental_rerun()
152
+ # else:
153
+ # DataInference().view(
154
+ # DataInference.Model(),
155
+ # st.session_state["ui_width"],
156
+ # st.session_state["device_type"],
157
+ # st.session_state["device_width"],
158
+ # )
159
+
160
+ # logout_widget()
161
+
162
+ # if menuItem == model.option6:
163
+ # if (
164
+ # "ui_width" not in st.session_state
165
+ # or "device_type" not in st.session_state
166
+ # or "device_width" not in st.session_state
167
+ # ):
168
+ # # Get UI width
169
+ # ui_width = st_js.st_javascript("window.innerWidth", key="ui_width_comp")
170
+ # device_width = st_js.st_javascript(
171
+ # "window.screen.width", key="device_width_comp"
172
+ # )
173
+
174
+ # if ui_width > 0 and device_width > 0:
175
+ # # Add 20% of current screen width to compensate for the sidebar
176
+ # ui_width = round(ui_width + (20 * ui_width / 100))
177
+
178
+ # if device_width > 768:
179
+ # device_type = "desktop"
180
+ # else:
181
+ # device_type = "mobile"
182
+
183
+ # st.session_state["ui_width"] = ui_width
184
+ # st.session_state["device_type"] = device_type
185
+ # st.session_state["device_width"] = device_width
186
+
187
+ # st.experimental_rerun()
188
+ # else:
189
+ # DataReview().view(
190
+ # DataReview.Model(),
191
+ # st.session_state["ui_width"],
192
+ # st.session_state["device_type"],
193
+ # st.session_state["device_width"],
194
+ # )
195
+
196
+ # logout_widget()
197
+
198
+ if menuItem == model.option7:
199
+ Setup().view(Setup.Model())
200
+ logout_widget()
201
+
202
+ # if menuItem == model.option8:
203
+ # About().view(About.Model())
204
+ # logout_widget()
205
+
206
+
207
+ def logout_widget():
208
+ with st.sidebar:
209
+ st.markdown("---")
210
+ # st.write("User:", "John Doe")
211
+ st.write("Version:", "2.0.0")
212
+ # st.button("Logout")
213
+ # st.markdown("---")
214
+
215
+ if "visitors" not in st.session_state:
216
+ with open("docs/visitors.json", "r") as f:
217
+ visitors_json = json.load(f)
218
+ visitors = visitors_json["meta"]["visitors"]
219
+
220
+ visitors += 1
221
+ visitors_json["meta"]["visitors"] = visitors
222
+
223
+ with open("docs/visitors.json", "w") as f:
224
+ json.dump(visitors_json, f)
225
+
226
+ st.session_state["visitors"] = visitors
227
+ else:
228
+ visitors = st.session_state["visitors"]
229
+
230
+ st.write("Counter:", visitors)
231
+
232
+
233
+ view(Model())
requirements.txt ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ streamlit==1.22.0
2
+ streamlit_option_menu==0.3.6
3
+ streamlit-sparrow-labeling==0.1.1
4
+ streamlit_nested_layout==0.1.1
5
+ streamlit-javascript==0.1.5
6
+ natsort
7
+ streamlit-aggrid==0.3.3
8
+ pandas<2.0
9
+ numpy<2.0
toolbar/__init__.py ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ import streamlit.components.v1 as components
2
+ component_toolbar_buttons = components.declare_component(
3
+ name='component_toolbar_buttons',
4
+ path='./toolbar'
5
+ )
toolbar/index.html ADDED
@@ -0,0 +1,150 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <html>
2
+ <head>
3
+ <link href="https://cdnjs.cloudflare.com/ajax/libs/flowbite/1.6.3/flowbite.min.css" rel="stylesheet" />
4
+ </head>
5
+
6
+ <!--
7
+ ----------------------------------------------------
8
+ Your custom static HTML goes in the body:
9
+ -->
10
+
11
+ <body>
12
+ <div class="inline-flex rounded-md shadow-sm" role="group">
13
+ <button id="create" type="button" onclick="call(this)" class="inline-flex items-center px-4 py-2 text-sm font-medium text-gray-900 bg-white border border-gray-200 rounded-l-lg hover:bg-gray-100 hover:text-blue-700 focus:z-10 focus:ring-2 focus:ring-blue-700 focus:text-blue-700 dark:bg-gray-700 dark:border-gray-600 dark:text-white dark:hover:text-white dark:hover:bg-gray-600 dark:focus:ring-blue-500 dark:focus:text-white">
14
+ <svg aria-hidden="true" class="w-4 h-4 mr-2 fill-current" fill="currentColor" viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg">
15
+ <path fill-rule="evenodd" d="M12 3.75a.75.75 0 01.75.75v6.75h6.75a.75.75 0 010 1.5h-6.75v6.75a.75.75 0 01-1.5 0v-6.75H4.5a.75.75 0 010-1.5h6.75V4.5a.75.75 0 01.75-.75z" clip-rule="evenodd"></path>
16
+ </svg>
17
+ Create
18
+ </button>
19
+ <button id="delete" type="button" onclick="call(this)" class="inline-flex items-center px-4 py-2 text-sm font-medium text-gray-900 bg-white border-t border-b border-gray-200 hover:bg-gray-100 hover:text-blue-700 focus:z-10 focus:ring-2 focus:ring-blue-700 focus:text-blue-700 dark:bg-gray-700 dark:border-gray-600 dark:text-white dark:hover:text-white dark:hover:bg-gray-600 dark:focus:ring-blue-500 dark:focus:text-white">
20
+ <svg aria-hidden="true" class="w-4 h-4 mr-2 fill-current" fill="currentColor" viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg">
21
+ <path d="M6.75 9.25a.75.75 0 000 1.5h6.5a.75.75 0 000-1.5h-6.5z"></path>
22
+ </svg>
23
+ Delete
24
+ </button>
25
+ <button id="save" type="button" onclick="call(this)" class="inline-flex items-center px-4 py-2 text-sm font-medium text-gray-900 bg-white border border-gray-200 rounded-r-md hover:bg-gray-100 hover:text-blue-700 focus:z-10 focus:ring-2 focus:ring-blue-700 focus:text-blue-700 dark:bg-gray-700 dark:border-gray-600 dark:text-white dark:hover:text-white dark:hover:bg-gray-600 dark:focus:ring-blue-500 dark:focus:text-white">
26
+ <svg aria-hidden="true" class="w-4 h-4 mr-2 fill-current" fill="currentColor" viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg">
27
+ <path fill-rule="evenodd" d="M16.704 4.153a.75.75 0 01.143 1.052l-8 10.5a.75.75 0 01-1.127.075l-4.5-4.5a.75.75 0 011.06-1.06l3.894 3.893 7.48-9.817a.75.75 0 011.05-.143z" clip-rule="evenodd"></path>
28
+ </svg>
29
+ Save
30
+ </button>
31
+ </div>
32
+ </body>
33
+
34
+ <script type="text/javascript">
35
+ // ----------------------------------------------------
36
+ // Use these functions as is to perform required Streamlit
37
+ // component lifecycle actions:
38
+ //
39
+ // 1. Signal Streamlit client that component is ready
40
+ // 2. Signal Streamlit client to set visible height of the component
41
+ // (this is optional, in case Streamlit doesn't correctly auto-set it)
42
+ // 3. Pass values from component to Streamlit client
43
+ //
44
+
45
+ // Helper function to send type and data messages to Streamlit client
46
+
47
+ const SET_COMPONENT_VALUE = "streamlit:setComponentValue"
48
+ const RENDER = "streamlit:render"
49
+ const COMPONENT_READY = "streamlit:componentReady"
50
+ const SET_FRAME_HEIGHT = "streamlit:setFrameHeight"
51
+
52
+ function _sendMessage(type, data) {
53
+ // copy data into object
54
+ var outboundData = Object.assign({
55
+ isStreamlitMessage: true,
56
+ type: type,
57
+ }, data)
58
+
59
+ if (type == SET_COMPONENT_VALUE) {
60
+ console.log("_sendMessage data: " + JSON.stringify(data))
61
+ console.log("_sendMessage outboundData: " + JSON.stringify(outboundData))
62
+ }
63
+
64
+ window.parent.postMessage(outboundData, "*")
65
+ }
66
+
67
+ function initialize(pipeline) {
68
+
69
+ // Hook Streamlit's message events into a simple dispatcher of pipeline handlers
70
+ window.addEventListener("message", (event) => {
71
+ if (event.data.type == RENDER) {
72
+ // The event.data.args dict holds any JSON-serializable value
73
+ // sent from the Streamlit client. It is already deserialized.
74
+ pipeline.forEach(handler => {
75
+ handler(event.data.args)
76
+ })
77
+ }
78
+ })
79
+
80
+ _sendMessage(COMPONENT_READY, { apiVersion: 1 });
81
+
82
+ // Component should be mounted by Streamlit in an iframe, so try to autoset the iframe height.
83
+ window.addEventListener("load", () => {
84
+ window.setTimeout(function () {
85
+ setFrameHeight(document.documentElement.clientHeight)
86
+ }, 0)
87
+ })
88
+
89
+ // Optionally, if auto-height computation fails, you can manually set it
90
+ // (uncomment below)
91
+ //setFrameHeight(200)
92
+ }
93
+
94
+ function setFrameHeight(height) {
95
+ _sendMessage(SET_FRAME_HEIGHT, { height: height })
96
+ }
97
+
98
+ // The `data` argument can be any JSON-serializable value.
99
+ function notifyHost(data) {
100
+ _sendMessage(SET_COMPONENT_VALUE, data)
101
+ }
102
+
103
+ // ----------------------------------------------------
104
+ // Your custom functionality for the component goes here:
105
+
106
+ function call(button) {
107
+ timestamp = Date.now()
108
+ action = {
109
+ "action": button.id,
110
+ "timestamp": timestamp
111
+ }
112
+ notifyHost({
113
+ value: action,
114
+ dataType: "json",
115
+ })
116
+ }
117
+
118
+ // ----------------------------------------------------
119
+ // Here you can customize a pipeline of handlers for
120
+ // inbound properties from the Streamlit client app
121
+
122
+ // Set initial value sent from Streamlit!
123
+ function initializeProps_Handler(props) {
124
+ for (let key of Object.keys(props.buttons)) {
125
+ btn = document.getElementById(key)
126
+ btn.disabled = props.buttons[key]
127
+ }
128
+
129
+ // btn.disabled = props.initial_state.delete_disabled
130
+ }
131
+ // Access values sent from Streamlit!
132
+ function dataUpdate_Handler(props) {
133
+ // let msgLabel = document.getElementById("message_label")
134
+ // msgLabel.innerText = `Update [${props.counter}] at ${props.datetime}`
135
+ }
136
+ // Simply log received data dictionary
137
+ function log_Handler(props) {
138
+ console.log("Received from Streamlit: " + JSON.stringify(props))
139
+ }
140
+
141
+ let pipeline = [initializeProps_Handler, dataUpdate_Handler, log_Handler]
142
+
143
+ // ----------------------------------------------------
144
+ // Finally, initialize component passing in pipeline
145
+
146
+ initialize(pipeline)
147
+
148
+ </script>
149
+
150
+ </html>
toolbar_main/__init__.py ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ import streamlit.components.v1 as components
2
+ component_toolbar_main = components.declare_component(
3
+ name='component_toolbar_main',
4
+ path='./toolbar_main'
5
+ )
toolbar_main/index.html ADDED
@@ -0,0 +1,149 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <html>
2
+ <head>
3
+ <link href="https://cdnjs.cloudflare.com/ajax/libs/flowbite/1.6.3/flowbite.min.css" rel="stylesheet" />
4
+ </head>
5
+
6
+ <!--
7
+ ----------------------------------------------------
8
+ Your custom static HTML goes in the body:
9
+ -->
10
+
11
+ <body>
12
+ <div class="inline-flex rounded-md shadow-sm" role="group">
13
+ <button id="up" type="button" onclick="call(this)" class="inline-flex items-center px-4 py-2 text-sm font-medium text-gray-900 bg-white border border-gray-200 rounded-l-lg hover:bg-gray-100 hover:text-blue-700 focus:z-10 focus:ring-2 focus:ring-blue-700 focus:text-blue-700 dark:bg-gray-700 dark:border-gray-600 dark:text-white dark:hover:text-white dark:hover:bg-gray-600 dark:focus:ring-blue-500 dark:focus:text-white">
14
+ <svg aria-hidden="true" class="w-4 h-4 mr-2 fill-current" fill="currentColor" viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg">
15
+ <path fill-rule="evenodd" d="M4.5 15.75l7.5-7.5 7.5 7.5" clip-rule="evenodd"></path>
16
+ </svg>
17
+ Up
18
+ </button>
19
+ <button id="down" type="button" onclick="call(this)" class="inline-flex items-center px-4 py-2 text-sm font-medium text-gray-900 bg-white border-t border-b border-gray-200 hover:bg-gray-100 hover:text-blue-700 focus:z-10 focus:ring-2 focus:ring-blue-700 focus:text-blue-700 dark:bg-gray-700 dark:border-gray-600 dark:text-white dark:hover:text-white dark:hover:bg-gray-600 dark:focus:ring-blue-500 dark:focus:text-white">
20
+ <svg aria-hidden="true" class="w-4 h-4 mr-2 fill-current" fill="currentColor" viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg">
21
+ <path d="M19.5 8.25l-7.5 7.5-7.5-7.5"></path>
22
+ </svg>
23
+ Down
24
+ </button>
25
+ <button id="save" type="button" onclick="call(this)" class="inline-flex items-center px-4 py-2 text-sm font-medium text-gray-900 bg-white border border-gray-200 rounded-r-md hover:bg-gray-100 hover:text-blue-700 focus:z-10 focus:ring-2 focus:ring-blue-700 focus:text-blue-700 dark:bg-gray-700 dark:border-gray-600 dark:text-white dark:hover:text-white dark:hover:bg-gray-600 dark:focus:ring-blue-500 dark:focus:text-white">
26
+ <svg aria-hidden="true" class="w-4 h-4 mr-2 fill-current" fill="currentColor" viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg">
27
+ <path fill-rule="evenodd" d="M16.704 4.153a.75.75 0 01.143 1.052l-8 10.5a.75.75 0 01-1.127.075l-4.5-4.5a.75.75 0 011.06-1.06l3.894 3.893 7.48-9.817a.75.75 0 011.05-.143z" clip-rule="evenodd"></path>
28
+ </svg>
29
+ Save
30
+ </button>
31
+ </div>
32
+ </body>
33
+
34
+ <script type="text/javascript">
35
+ // ----------------------------------------------------
36
+ // Use these functions as is to perform required Streamlit
37
+ // component lifecycle actions:
38
+ //
39
+ // 1. Signal Streamlit client that component is ready
40
+ // 2. Signal Streamlit client to set visible height of the component
41
+ // (this is optional, in case Streamlit doesn't correctly auto-set it)
42
+ // 3. Pass values from component to Streamlit client
43
+ //
44
+
45
+ // Helper function to send type and data messages to Streamlit client
46
+
47
+ const SET_COMPONENT_VALUE = "streamlit:setComponentValue"
48
+ const RENDER = "streamlit:render"
49
+ const COMPONENT_READY = "streamlit:componentReady"
50
+ const SET_FRAME_HEIGHT = "streamlit:setFrameHeight"
51
+
52
+ function _sendMessage(type, data) {
53
+ // copy data into object
54
+ var outboundData = Object.assign({
55
+ isStreamlitMessage: true,
56
+ type: type,
57
+ }, data)
58
+
59
+ if (type == SET_COMPONENT_VALUE) {
60
+ console.log("_sendMessage data: " + JSON.stringify(data))
61
+ console.log("_sendMessage outboundData: " + JSON.stringify(outboundData))
62
+ }
63
+
64
+ window.parent.postMessage(outboundData, "*")
65
+ }
66
+
67
+ function initialize(pipeline) {
68
+
69
+ // Hook Streamlit's message events into a simple dispatcher of pipeline handlers
70
+ window.addEventListener("message", (event) => {
71
+ if (event.data.type == RENDER) {
72
+ // The event.data.args dict holds any JSON-serializable value
73
+ // sent from the Streamlit client. It is already deserialized.
74
+ pipeline.forEach(handler => {
75
+ handler(event.data.args)
76
+ })
77
+ }
78
+ })
79
+
80
+ _sendMessage(COMPONENT_READY, { apiVersion: 1 });
81
+
82
+ // Component should be mounted by Streamlit in an iframe, so try to autoset the iframe height.
83
+ window.addEventListener("load", () => {
84
+ window.setTimeout(function () {
85
+ setFrameHeight(document.documentElement.clientHeight)
86
+ }, 0)
87
+ })
88
+
89
+ // Optionally, if auto-height computation fails, you can manually set it
90
+ // (uncomment below)
91
+ //setFrameHeight(200)
92
+ }
93
+
94
+ function setFrameHeight(height) {
95
+ _sendMessage(SET_FRAME_HEIGHT, { height: height })
96
+ }
97
+
98
+ // The `data` argument can be any JSON-serializable value.
99
+ function notifyHost(data) {
100
+ _sendMessage(SET_COMPONENT_VALUE, data)
101
+ }
102
+
103
+ // ----------------------------------------------------
104
+ // Your custom functionality for the component goes here:
105
+
106
+ function call(button) {
107
+ timestamp = Date.now()
108
+ action = {
109
+ "action": button.id,
110
+ "timestamp": timestamp
111
+ }
112
+ notifyHost({
113
+ value: action,
114
+ dataType: "json",
115
+ })
116
+ }
117
+
118
+ // ----------------------------------------------------
119
+ // Here you can customize a pipeline of handlers for
120
+ // inbound properties from the Streamlit client app
121
+
122
+ // Set initial value sent from Streamlit!
123
+ function initializeProps_Handler(props) {
124
+ for (let key of Object.keys(props.buttons)) {
125
+ btn = document.getElementById(key)
126
+ btn.disabled = props.buttons[key]['disabled']
127
+ btn.style.display = props.buttons[key]['rendered']
128
+ }
129
+ }
130
+ // Access values sent from Streamlit!
131
+ function dataUpdate_Handler(props) {
132
+ // let msgLabel = document.getElementById("message_label")
133
+ // msgLabel.innerText = `Update [${props.counter}] at ${props.datetime}`
134
+ }
135
+ // Simply log received data dictionary
136
+ function log_Handler(props) {
137
+ console.log("Received from Streamlit: " + JSON.stringify(props))
138
+ }
139
+
140
+ let pipeline = [initializeProps_Handler, dataUpdate_Handler, log_Handler]
141
+
142
+ // ----------------------------------------------------
143
+ // Finally, initialize component passing in pipeline
144
+
145
+ initialize(pipeline)
146
+
147
+ </script>
148
+
149
+ </html>
tools/agstyler.py ADDED
@@ -0,0 +1,77 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # adjusted from: https://github.com/nryabykh/streamlit-aggrid-hints
2
+
3
+ from st_aggrid import AgGrid
4
+ from st_aggrid.grid_options_builder import GridOptionsBuilder
5
+ from st_aggrid.shared import GridUpdateMode, JsCode
6
+
7
+
8
+ def get_numeric_style_with_precision(precision: int) -> dict:
9
+ return {"type": ["numericColumn", "customNumericFormat"], "precision": precision}
10
+
11
+
12
+ PRECISION_ZERO = get_numeric_style_with_precision(0)
13
+ PRECISION_ONE = get_numeric_style_with_precision(1)
14
+ PRECISION_TWO = get_numeric_style_with_precision(2)
15
+ PINLEFT = {"pinned": "left"}
16
+
17
+
18
+ def draw_grid(
19
+ df,
20
+ formatter: dict = None,
21
+ selection="multiple",
22
+ use_checkbox=False,
23
+ fit_columns=False,
24
+ pagination_size=0,
25
+ theme="streamlit",
26
+ wrap_text: bool = False,
27
+ auto_height: bool = False,
28
+ grid_options: dict = None,
29
+ key=None,
30
+ css: dict = None
31
+ ):
32
+
33
+ gb = GridOptionsBuilder()
34
+ gb.configure_default_column(
35
+ filterable=True,
36
+ groupable=False,
37
+ editable=False,
38
+ wrapText=wrap_text,
39
+ autoHeight=auto_height
40
+ )
41
+
42
+ if grid_options is not None:
43
+ gb.configure_grid_options(**grid_options)
44
+
45
+ for latin_name, (cyr_name, style_dict) in formatter.items():
46
+ gb.configure_column(latin_name, header_name=cyr_name, **style_dict)
47
+
48
+ gb.configure_selection(selection_mode=selection, use_checkbox=use_checkbox)
49
+
50
+ if pagination_size > 0:
51
+ gb.configure_pagination(enabled=True, paginationAutoPageSize=False, paginationPageSize=pagination_size)
52
+
53
+ return AgGrid(
54
+ df,
55
+ gridOptions=gb.build(),
56
+ update_mode=GridUpdateMode.SELECTION_CHANGED | GridUpdateMode.VALUE_CHANGED,
57
+ allow_unsafe_jscode=True,
58
+ fit_columns_on_grid_load=fit_columns,
59
+ theme=theme,
60
+ key=key,
61
+ custom_css=css,
62
+ enable_enterprise_modules=False
63
+ )
64
+
65
+
66
+ def highlight(color, condition):
67
+ code = f"""
68
+ function(params) {{
69
+ color = "{color}";
70
+ if ({condition}) {{
71
+ return {{
72
+ 'backgroundColor': color
73
+ }}
74
+ }}
75
+ }};
76
+ """
77
+ return JsCode(code)
tools/data_review.py ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from natsort import natsorted
3
+ import json
4
+
5
+
6
+ def annotation_review():
7
+ # get list of files in json directory
8
+ processed_file_names = get_processed_file_names('../docs/json/')
9
+ for file_name in processed_file_names:
10
+ # open json file
11
+ with open('../docs/json/' + file_name + '.json') as json_file:
12
+ json_file_data = json.load(json_file)
13
+ version = json_file_data['meta']['version']
14
+ if version == "v0.1":
15
+ print(file_name + " is v0.1")
16
+
17
+ def get_processed_file_names(dir_name):
18
+ # get ordered list of files without file extension, excluding hidden files, with JSON extension only
19
+ file_names = [os.path.splitext(f)[0] for f in os.listdir(dir_name) if
20
+ os.path.isfile(os.path.join(dir_name, f)) and not f.startswith('.') and f.endswith('.json')]
21
+ file_names = natsorted(file_names)
22
+ return file_names
23
+
24
+ def main():
25
+ annotation_review()
26
+
27
+
28
+ if __name__ == '__main__':
29
+ main()
tools/st_functions.py ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+
3
+
4
+ def st_button(icon, url, label, iconsize):
5
+ if icon == 'youtube':
6
+ button_code = f'''
7
+ <p>
8
+ <a href={url} class="btn btn-outline-primary btn-lg btn-block" type="button" aria-pressed="true">
9
+ <svg xmlns="http://www.w3.org/2000/svg" width={iconsize} height={iconsize} fill="currentColor" class="bi bi-youtube" viewBox="0 0 16 16">
10
+ <path d="M8.051 1.999h.089c.822.003 4.987.033 6.11.335a2.01 2.01 0 0 1 1.415 1.42c.101.38.172.883.22 1.402l.01.104.022.26.008.104c.065.914.073 1.77.074 1.957v.075c-.001.194-.01 1.108-.082 2.06l-.008.105-.009.104c-.05.572-.124 1.14-.235 1.558a2.007 2.007 0 0 1-1.415 1.42c-1.16.312-5.569.334-6.18.335h-.142c-.309 0-1.587-.006-2.927-.052l-.17-.006-.087-.004-.171-.007-.171-.007c-1.11-.049-2.167-.128-2.654-.26a2.007 2.007 0 0 1-1.415-1.419c-.111-.417-.185-.986-.235-1.558L.09 9.82l-.008-.104A31.4 31.4 0 0 1 0 7.68v-.123c.002-.215.01-.958.064-1.778l.007-.103.003-.052.008-.104.022-.26.01-.104c.048-.519.119-1.023.22-1.402a2.007 2.007 0 0 1 1.415-1.42c.487-.13 1.544-.21 2.654-.26l.17-.007.172-.006.086-.003.171-.007A99.788 99.788 0 0 1 7.858 2h.193zM6.4 5.209v4.818l4.157-2.408L6.4 5.209z"/>
11
+ </svg>
12
+ {label}
13
+ </a>
14
+ </p>'''
15
+ elif icon == 'twitter':
16
+ button_code = f'''
17
+ <p>
18
+ <a href={url} class="btn btn-outline-primary btn-lg btn-block" type="button" aria-pressed="true">
19
+ <svg xmlns="http://www.w3.org/2000/svg" width={iconsize} height={iconsize} fill="currentColor" class="bi bi-twitter" viewBox="0 0 16 16">
20
+ <path d="M5.026 15c6.038 0 9.341-5.003 9.341-9.334 0-.14 0-.282-.006-.422A6.685 6.685 0 0 0 16 3.542a6.658 6.658 0 0 1-1.889.518 3.301 3.301 0 0 0 1.447-1.817 6.533 6.533 0 0 1-2.087.793A3.286 3.286 0 0 0 7.875 6.03a9.325 9.325 0 0 1-6.767-3.429 3.289 3.289 0 0 0 1.018 4.382A3.323 3.323 0 0 1 .64 6.575v.045a3.288 3.288 0 0 0 2.632 3.218 3.203 3.203 0 0 1-.865.115 3.23 3.23 0 0 1-.614-.057 3.283 3.283 0 0 0 3.067 2.277A6.588 6.588 0 0 1 .78 13.58a6.32 6.32 0 0 1-.78-.045A9.344 9.344 0 0 0 5.026 15z"/>
21
+ </svg>
22
+ {label}
23
+ </a>
24
+ </p>'''
25
+ elif icon == 'linkedin':
26
+ button_code = f'''
27
+ <p>
28
+ <a href={url} class="btn btn-outline-primary btn-lg btn-block" type="button" aria-pressed="true">
29
+ <svg xmlns="http://www.w3.org/2000/svg" width={iconsize} height={iconsize} fill="currentColor" class="bi bi-linkedin" viewBox="0 0 16 16">
30
+ <path d="M0 1.146C0 .513.526 0 1.175 0h13.65C15.474 0 16 .513 16 1.146v13.708c0 .633-.526 1.146-1.175 1.146H1.175C.526 16 0 15.487 0 14.854V1.146zm4.943 12.248V6.169H2.542v7.225h2.401zm-1.2-8.212c.837 0 1.358-.554 1.358-1.248-.015-.709-.52-1.248-1.342-1.248-.822 0-1.359.54-1.359 1.248 0 .694.521 1.248 1.327 1.248h.016zm4.908 8.212V9.359c0-.216.016-.432.08-.586.173-.431.568-.878 1.232-.878.869 0 1.216.662 1.216 1.634v3.865h2.401V9.25c0-2.22-1.184-3.252-2.764-3.252-1.274 0-1.845.7-2.165 1.193v.025h-.016a5.54 5.54 0 0 1 .016-.025V6.169h-2.4c.03.678 0 7.225 0 7.225h2.4z"/>
31
+ </svg>
32
+ {label}
33
+ </a>
34
+ </p>'''
35
+ elif icon == 'medium':
36
+ button_code = f'''
37
+ <p>
38
+ <a href={url} class="btn btn-outline-primary btn-lg btn-block" type="button" aria-pressed="true">
39
+ <svg xmlns="http://www.w3.org/2000/svg" width={iconsize} height={iconsize} fill="currentColor" class="bi bi-medium" viewBox="0 0 16 16">
40
+ <path d="M9.025 8c0 2.485-2.02 4.5-4.513 4.5A4.506 4.506 0 0 1 0 8c0-2.486 2.02-4.5 4.512-4.5A4.506 4.506 0 0 1 9.025 8zm4.95 0c0 2.34-1.01 4.236-2.256 4.236-1.246 0-2.256-1.897-2.256-4.236 0-2.34 1.01-4.236 2.256-4.236 1.246 0 2.256 1.897 2.256 4.236zM16 8c0 2.096-.355 3.795-.794 3.795-.438 0-.793-1.7-.793-3.795 0-2.096.355-3.795.794-3.795.438 0 .793 1.699.793 3.795z"/>
41
+ </svg>
42
+ {label}
43
+ </a>
44
+ </p>'''
45
+ elif icon == 'newsletter':
46
+ button_code = f'''
47
+ <p>
48
+ <a href={url} class="btn btn-outline-primary btn-lg btn-block" type="button" aria-pressed="true">
49
+ <svg xmlns="http://www.w3.org/2000/svg" width={iconsize} height={iconsize} fill="currentColor" class="bi bi-envelope" viewBox="0 0 16 16">
50
+ <path d="M0 4a2 2 0 0 1 2-2h12a2 2 0 0 1 2 2v8a2 2 0 0 1-2 2H2a2 2 0 0 1-2-2V4Zm2-1a1 1 0 0 0-1 1v.217l7 4.2 7-4.2V4a1 1 0 0 0-1-1H2Zm13 2.383-4.708 2.825L15 11.105V5.383Zm-.034 6.876-5.64-3.471L8 9.583l-1.326-.795-5.64 3.47A1 1 0 0 0 2 13h12a1 1 0 0 0 .966-.741ZM1 11.105l4.708-2.897L1 5.383v5.722Z"/>
51
+ </svg>
52
+ {label}
53
+ </a>
54
+ </p>'''
55
+ elif icon == 'github':
56
+ button_code = f'''
57
+ <p>
58
+ <a href={url} class="btn btn-outline-primary btn-lg btn-block" type="button" aria-pressed="true">
59
+ <svg xmlns="http://www.w3.org/2000/svg" width={iconsize} height={iconsize} fill="currentColor" class="bi bi-github" viewBox="0 0 16 16">
60
+ <path d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.012 8.012 0 0 0 16 8c0-4.42-3.58-8-8-8z"/>
61
+ </svg>
62
+ {label}
63
+ </a>
64
+ </p>'''
65
+ elif icon == '':
66
+ button_code = f'''
67
+ <p>
68
+ <a href={url} class="btn btn-outline-primary btn-lg btn-block" type="button" aria-pressed="true">
69
+ {label}
70
+ </a>
71
+ </p>'''
72
+ return st.markdown(button_code, unsafe_allow_html=True)
tools/style.css ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /* Move block container higher */
2
+ div.block-container.css-z5fcl4.egzxvld4 {
3
+ margin-top: -5em;
4
+ }
5
+
6
+ /* Move menu container higher */
7
+ div.css-1544g2n.e1fqkh3o4 {
8
+ padding-top: 3rem;
9
+ }
10
+
11
+ /* Hide anchor link */
12
+ .css-1dgmtll.e16nr0p32 {
13
+ display: none
14
+ }
15
+
16
+ div[data-testid="metric-container"] {
17
+ background-color: #f7f8fa;
18
+ border: 1px solid #0c0d0d;
19
+ padding: 5% 5% 5% 10%;
20
+ border-radius: 5px;
21
+ color: rgb(30, 103, 119);
22
+ overflow-wrap: break-word;
23
+ }
24
+
25
+ /* breakline for metric text */
26
+ div[data-testid="metric-container"] > label[data-testid="stMetricLabel"] > div {
27
+ overflow-wrap: break-word;
28
+ white-space: break-spaces;
29
+ color: black;
30
+ }
31
+
32
+ /* Hide Streamlit bars */
33
+ #MainMenu {
34
+ visibility: hidden;
35
+ }
36
+
37
+ footer {
38
+ visibility: hidden;
39
+ }
40
+
41
+ /*header {*/
42
+ /* visibility: hidden;*/
43
+ /*}*/
44
+
45
+ /*About page styling*/
46
+
47
+ .css-1v0mbdj.etr89bj1 {
48
+ display: block;
49
+ margin-left: auto;
50
+ margin-right: auto;
51
+ min-width: 180px;
52
+ max-width: 40%;
53
+ }
tools/utilities.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+
3
+
4
+ def load_css():
5
+ with open("tools/style.css") as f:
6
+ st.markdown('<style>{}</style>'.format(f.read()), unsafe_allow_html=True)
7
+ st.markdown(
8
+ '<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">',
9
+ unsafe_allow_html=True)
views/about.py ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+ from PIL import Image
3
+ from tools.st_functions import st_button
4
+
5
+
6
+ class About:
7
+ class Model:
8
+ pageTitle = "About"
9
+
10
+ def view(self, model):
11
+ # st.title(model.pageTitle)
12
+
13
+ st.write(
14
+ "[![Star](https://img.shields.io/github/stars/katanaml/sparrow.svg?logo=github&style=social)](https://github.com/katanaml/sparrow)")
15
+
16
+ col1, col2, col3 = st.columns(3)
17
+ col2.image(Image.open('assets/ab.png'))
18
+
19
+ st.markdown("<h1 style='text-align: center; color: black; font-weight: bold;'>Andrej Baranovskij, Founder Katana ML</h1>",
20
+ unsafe_allow_html=True)
21
+
22
+ st.info(
23
+ 'Sparrow is a tool for data extraction from PDFs, images, and other documents. It is a part of Katana ML, '
24
+ 'a platform for data science and machine learning.')
25
+
26
+ icon_size = 20
27
+
28
+ st_button('youtube', 'https://www.youtube.com/@AndrejBaranovskij', 'Andrej Baranovskij YouTube channel', icon_size)
29
+ st_button('github', 'https://github.com/katanaml/sparrow', 'Sparrow GitHub', icon_size)
30
+ st_button('twitter', 'https://twitter.com/andrejusb', 'Follow me on Twitter', icon_size)
31
+ st_button('medium', 'https://andrejusb.medium.com', 'Read my Blogs on Medium', icon_size)
32
+ st_button('linkedin', 'https://www.linkedin.com/in/andrej-baranovskij/', 'Follow me on LinkedIn', icon_size)
33
+ st_button('', 'https://katanaml.io', 'Katana ML', icon_size)
views/dashboard.py ADDED
@@ -0,0 +1,378 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+ import numpy as np
3
+ import pandas as pd
4
+ import json
5
+ import altair as alt
6
+ from pathlib import Path
7
+ import requests
8
+
9
+
10
+ class Dashboard:
11
+ class Model:
12
+ pageTitle = "Dashboard"
13
+
14
+ wordsTitle = "Words"
15
+
16
+ inferenceTimeTitle = "Inference Time"
17
+
18
+ documentsTitle = "Documents"
19
+
20
+ dailyInferenceTitle = "Top Daily Inference"
21
+
22
+ accuracyTitle = "Mean Accuracy"
23
+
24
+ titleModelEval = "## Evaluation Accuracy"
25
+ titleInferencePerformance = "## Inference Performance"
26
+ titleDatasetInfo = "## Dataset Info"
27
+ titleDataAnnotation = "## Data Annotation"
28
+ titleTrainingPerformance = "## Training Performance"
29
+ titleEvaluationPerformance = "## Evaluation Performance"
30
+
31
+ status_file = "docs/status.json"
32
+ annotation_files_dir = "docs/json"
33
+
34
+ def view(self, model):
35
+ # st.title(model.pageTitle)
36
+
37
+ api_url = "https://katanaml-org-sparrow-ml.hf.space/api-inference/v1/sparrow-ml/statistics"
38
+ json_data_inference = []
39
+ response = requests.get(api_url)
40
+ if response.status_code == 200:
41
+ json_data_inference = response.json()
42
+ else:
43
+ print(f"Error: Unable to fetch data from the API (status code {response.status_code})")
44
+
45
+ api_url_t = "https://katanaml-org-sparrow-ml.hf.space/api-training/v1/sparrow-ml/statistics/training"
46
+ json_data_training = []
47
+ response_t = requests.get(api_url_t)
48
+ if response_t.status_code == 200:
49
+ json_data_training = response_t.json()
50
+ else:
51
+ print(f"Error: Unable to fetch data from the API (status code {response_t.status_code})")
52
+
53
+ api_url_e = "https://katanaml-org-sparrow-ml.hf.space/api-training/v1/sparrow-ml/statistics/evaluate"
54
+ json_data_evaluate = []
55
+ response_e = requests.get(api_url_e)
56
+ if response_e.status_code == 200:
57
+ json_data_evaluate = response_e.json()
58
+ else:
59
+ print(f"Error: Unable to fetch data from the API (status code {response_e.status_code})")
60
+
61
+ with st.container():
62
+ col1, col2, col3, col4, col5 = st.columns(5)
63
+
64
+ with col1:
65
+ words_count = 0
66
+ delta_words = 0
67
+
68
+ if len(json_data_inference) > 3:
69
+ for i in range(0, len(json_data_inference)):
70
+ words_count = words_count + json_data_inference[i][1]
71
+
72
+ avg_word_count = words_count / len(json_data_inference)
73
+ avg_word_last = (json_data_inference[len(json_data_inference) - 1][1]
74
+ + json_data_inference[len(json_data_inference) - 2][1] +
75
+ json_data_inference[len(json_data_inference) - 3][1]) / 3
76
+
77
+ if avg_word_last >= avg_word_count:
78
+ delta_words = round(100 - ((avg_word_count * 100) / avg_word_last), 2)
79
+ else:
80
+ delta_words = round(100 - ((avg_word_last * 100) / avg_word_count), 2) * -1
81
+
82
+ words_count = words_count / 1000
83
+ st.metric(label=model.wordsTitle, value=str(words_count) + 'K', delta=str(delta_words) + "%")
84
+
85
+ with col2:
86
+ docs_count = len(json_data_inference)
87
+ delta_docs = 0
88
+
89
+ if docs_count > 3:
90
+ inference_dates = []
91
+ for i in range(0, len(json_data_inference)):
92
+ inference_dates.append(json_data_inference[i][4].split(" ")[0])
93
+
94
+ inference_dates_unique = []
95
+ for item in inference_dates:
96
+ if item not in inference_dates_unique:
97
+ inference_dates_unique.append(item)
98
+
99
+ if len(inference_dates_unique) > 3:
100
+ inference_dates_dict = {}
101
+ for i, key in enumerate(inference_dates_unique):
102
+ inference_dates_dict[key] = [0]
103
+
104
+ for i in range(0, len(json_data_inference)):
105
+ inference_dates_dict[json_data_inference[i][4].split(" ")[0]][0] = \
106
+ inference_dates_dict[json_data_inference[i][4].split(" ")[0]][0] + 1
107
+
108
+ # calculate average for values from inference_dates_dict
109
+ avg_value = 0
110
+ for key, value in inference_dates_dict.items():
111
+ avg_value = avg_value + value[0]
112
+ avg_value = round(avg_value / len(inference_dates_dict), 2)
113
+
114
+ # calculate average for last 3 values from inference_dates_dict
115
+ avg_value_last = 0
116
+ for i in range(1, 4):
117
+ avg_value_last = avg_value_last + inference_dates_dict[inference_dates_unique[len(inference_dates_unique) - i]][0]
118
+ avg_value_last = round(avg_value_last / 3, 2)
119
+
120
+ if avg_value_last > avg_value:
121
+ delta_docs = round(100 - ((avg_value * 100) / avg_value_last), 2)
122
+ else:
123
+ delta_docs = round(100 - ((avg_value_last * 100) / avg_value), 2) * -1
124
+
125
+ st.metric(label=model.documentsTitle, value=docs_count, delta=str(delta_docs) + "%")
126
+
127
+ with col3:
128
+ inference_dates = []
129
+ for i in range(0, len(json_data_inference)):
130
+ inference_dates.append(json_data_inference[i][4].split(" ")[0])
131
+
132
+ inference_dates_unique = []
133
+ for item in inference_dates:
134
+ if item not in inference_dates_unique:
135
+ inference_dates_unique.append(item)
136
+
137
+ inference_dates_dict = {}
138
+ for i, key in enumerate(inference_dates_unique):
139
+ inference_dates_dict[key] = [0]
140
+
141
+ for i in range(0, len(json_data_inference)):
142
+ inference_dates_dict[json_data_inference[i][4].split(" ")[0]][0] = \
143
+ inference_dates_dict[json_data_inference[i][4].split(" ")[0]][0] + 1
144
+
145
+ # loop through the dictionary and find the max value
146
+ max_value = 0
147
+ for key, value in inference_dates_dict.items():
148
+ if value[0] > max_value:
149
+ max_value = value[0]
150
+
151
+ # calculate average for values from inference_dates_dict
152
+ avg_value = 0
153
+ for key, value in inference_dates_dict.items():
154
+ avg_value = avg_value + value[0]
155
+ avg_value = round(avg_value / len(inference_dates_dict), 2)
156
+
157
+ avg_delta = round(100 - ((avg_value * 100) / max_value), 2)
158
+
159
+ st.metric(label=model.dailyInferenceTitle, value=max_value, delta=str(avg_delta) + "%")
160
+
161
+ with col4:
162
+ inference_time_avg = 0
163
+
164
+ # calculate inference time average
165
+ for i in range(0, len(json_data_inference)):
166
+ inference_time_avg = inference_time_avg + json_data_inference[i][0]
167
+ inference_time_avg = round(inference_time_avg / len(json_data_inference), 2)
168
+
169
+ delta_time = 0
170
+ if len(json_data_inference) > 3:
171
+ avg_time_last = (json_data_inference[len(json_data_inference) - 1][0] +
172
+ json_data_inference[len(json_data_inference) - 2][0] +
173
+ json_data_inference[len(json_data_inference) - 3][0]) / 3
174
+
175
+ if avg_time_last > inference_time_avg:
176
+ delta_time = round(100 - ((inference_time_avg * 100) / avg_time_last), 2)
177
+ else:
178
+ delta_time = round(100 - ((avg_time_last * 100) / inference_time_avg), 2) * -1
179
+
180
+ st.metric(label=model.inferenceTimeTitle, value=str(inference_time_avg) + " s", delta=str(delta_time) + "%",
181
+ delta_color="inverse")
182
+
183
+ with col5:
184
+ models_unique = []
185
+ models_dict = {}
186
+ for i in range(0, len(json_data_evaluate)):
187
+ if json_data_evaluate[i][3] not in models_unique:
188
+ models_unique.append(json_data_evaluate[i][3])
189
+ models_dict[json_data_evaluate[i][3]] = json_data_evaluate[i][1]['mean_accuracy']
190
+
191
+ avg_accuracy = 0
192
+ for key, value in models_dict.items():
193
+ avg_accuracy = avg_accuracy + value
194
+ avg_accuracy = round(avg_accuracy / len(models_dict), 2)
195
+
196
+ if len(models_unique) > 3:
197
+ # calculate average accuracy for last 3 values
198
+ avg_accuracy_last = 0
199
+ for i in range(1, 4):
200
+ avg_accuracy_last = avg_accuracy_last + models_dict[models_unique[len(models_unique) - i]]
201
+ avg_accuracy_last = round(avg_accuracy_last / 3, 2)
202
+ else:
203
+ avg_accuracy_last = avg_accuracy
204
+
205
+ if avg_accuracy_last > avg_accuracy:
206
+ delta_accuracy = round(100 - ((avg_accuracy * 100) / avg_accuracy_last), 2)
207
+ else:
208
+ delta_accuracy = round(100 - ((avg_accuracy_last * 100) / avg_accuracy), 2) * -1
209
+
210
+ st.metric(label=model.accuracyTitle, value=avg_accuracy, delta=str(delta_accuracy) + "%",
211
+ delta_color="inverse")
212
+
213
+ st.markdown("---")
214
+
215
+
216
+ with st.container():
217
+ col1, col2 = st.columns(2)
218
+
219
+ with col1:
220
+ st.write(model.titleInferencePerformance)
221
+
222
+ models_dict = {}
223
+
224
+ models = []
225
+ for i in range(0, len(json_data_inference)):
226
+ models.append(json_data_inference[i][3])
227
+
228
+ models_unique = []
229
+ for item in models:
230
+ if item not in models_unique:
231
+ models_unique.append(item)
232
+
233
+ for i, key in enumerate(models_unique):
234
+ models_dict[key] = []
235
+
236
+ for i in range(0, len(json_data_inference)):
237
+ models_dict[json_data_inference[i][3]].append(round(json_data_inference[i][0]))
238
+
239
+ data = pd.DataFrame(models_dict)
240
+ st.line_chart(data)
241
+
242
+ with col2:
243
+ st.write(model.titleModelEval)
244
+
245
+ models_unique = []
246
+ models_dict = {}
247
+ for i in range(0, len(json_data_evaluate)):
248
+ if json_data_evaluate[i][3] not in models_unique:
249
+ models_unique.append(json_data_evaluate[i][3])
250
+ models_dict[json_data_evaluate[i][3]] = json_data_evaluate[i][1]['accuracies']
251
+
252
+ data = pd.DataFrame(models_dict)
253
+ st.line_chart(data)
254
+
255
+ st.markdown("---")
256
+
257
+ with st.container():
258
+ col1, col2, col3 = st.columns(3)
259
+
260
+ with col1:
261
+ with st.container():
262
+ st.write(model.titleDataAnnotation)
263
+
264
+ total, completed, in_progress = self.calculate_annotation_stats(model)
265
+
266
+ data = pd.DataFrame({"Status": ["Completed", "In Progress"], "Value": [completed, in_progress]})
267
+
268
+ # Create a horizontal bar chart
269
+ chart = alt.Chart(data).mark_bar().encode(
270
+ x='Value:Q',
271
+ y=alt.Y('Status:N', sort='-x'),
272
+ color=alt.Color('Status:N', legend=None)
273
+ )
274
+
275
+ st.altair_chart(chart)
276
+ with col2:
277
+ with st.container():
278
+ st.write(model.titleDatasetInfo)
279
+
280
+ api_url = "https://katanaml-org-sparrow-data.hf.space/api-dataset/v1/sparrow-data/dataset_info"
281
+
282
+ # Make the GET request
283
+ response = requests.get(api_url)
284
+
285
+ # Check if the request was successful (status code 200)
286
+ names = []
287
+ rows = []
288
+ if response.status_code == 200:
289
+ # Convert the response content to a JSON object
290
+ json_data = response.json()
291
+
292
+ for i in range(0, len(json_data['splits'])):
293
+ names.append(json_data['splits'][i]['name'])
294
+ rows.append(json_data['splits'][i]['number_of_rows'])
295
+ else:
296
+ print(f"Error: Unable to fetch data from the API (status code {response.status_code})")
297
+
298
+ data = pd.DataFrame({"Dataset": names, "Value": rows})
299
+
300
+ # Create a horizontal bar chart
301
+ chart = alt.Chart(data).mark_bar().encode(
302
+ x='Value:Q',
303
+ y=alt.Y('Dataset:N', sort='-x'),
304
+ color=alt.Color('Dataset:N', legend=None)
305
+ )
306
+
307
+ st.altair_chart(chart)
308
+ with col3:
309
+ with st.container():
310
+ st.write(model.titleTrainingPerformance)
311
+
312
+ models_dict = {}
313
+
314
+ for i in range(0, len(json_data_training)):
315
+ models_dict[i] = round(json_data_training[i][0])
316
+
317
+ data = pd.DataFrame({"Runs": models_dict.keys(), "Value": list(models_dict.values())})
318
+
319
+ # Create a horizontal bar chart
320
+ chart = alt.Chart(data).mark_bar().encode(
321
+ x='Value:Q',
322
+ y=alt.Y('Runs:N', sort='-x'),
323
+ color=alt.Color('Runs:N', legend=None)
324
+ )
325
+
326
+ st.altair_chart(chart)
327
+
328
+ st.markdown("---")
329
+
330
+ with st.container():
331
+ st.write(model.titleEvaluationPerformance)
332
+
333
+ runs_dict = {}
334
+
335
+ for i in range(0, len(json_data_evaluate)):
336
+ runs_dict[i] = round(json_data_evaluate[i][0])
337
+
338
+ data = pd.DataFrame({"Runs": runs_dict.keys(), "Value": list(runs_dict.values())})
339
+
340
+ # Create a horizontal bar chart
341
+ chart = alt.Chart(data).mark_bar().encode(
342
+ x='Value:Q',
343
+ y=alt.Y('Runs:N', sort='-x'),
344
+ color=alt.Color('Runs:N', legend=None)
345
+ )
346
+
347
+ st.altair_chart(chart)
348
+
349
+
350
+ def calculate_annotation_stats(self, model):
351
+ completed = 0
352
+ in_progress = 0
353
+ data_dir_path = Path(model.annotation_files_dir)
354
+
355
+ for file_name in data_dir_path.glob("*.json"):
356
+ with open(file_name, "r") as f:
357
+ data = json.load(f)
358
+ v = data['meta']['version']
359
+ if v == 'v0.1':
360
+ in_progress += 1
361
+ else:
362
+ completed += 1
363
+ total = completed + in_progress
364
+
365
+ status_json = {
366
+ "annotations": [
367
+ {
368
+ "completed": completed,
369
+ "in_progress": in_progress,
370
+ "total": total
371
+ }
372
+ ]
373
+ }
374
+
375
+ with open(model.status_file, "w") as f:
376
+ json.dump(status_json, f, indent=2)
377
+
378
+ return total, completed, in_progress
views/data_annotation.py ADDED
@@ -0,0 +1,883 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+ from PIL import Image
3
+ import streamlit_nested_layout
4
+ from streamlit_sparrow_labeling import st_sparrow_labeling
5
+ from streamlit_sparrow_labeling import DataProcessor
6
+ import json
7
+ import math
8
+ import os
9
+ from natsort import natsorted
10
+ from tools import agstyler
11
+ from tools.agstyler import PINLEFT
12
+ import pandas as pd
13
+ from toolbar_main import component_toolbar_main
14
+
15
+
16
+ class DataAnnotation:
17
+ class Model:
18
+ pageTitle = "Data Annotation"
19
+
20
+ img_file = None
21
+ rects_file = None
22
+ labels_file = "docs/labels.json"
23
+ groups_file = "docs/groups.json"
24
+ subgroup_file = "docs/subgroups.json"
25
+
26
+ assign_labels_text = "Assign Labels"
27
+ text_caption_1 = "Check 'Assign Labels' to enable editing of labels and values, move and resize the boxes to annotate the document."
28
+ text_caption_2 = "Add annotations by clicking and dragging on the document, when 'Assign Labels' is unchecked."
29
+
30
+ labels = [
31
+ "",
32
+ "invoice_no",
33
+ "invoice_date",
34
+ "seller",
35
+ "client",
36
+ "seller_tax_id",
37
+ "client_tax_id",
38
+ "iban",
39
+ "item_desc",
40
+ "item_qty",
41
+ "item_net_price",
42
+ "item_net_worth",
43
+ "item_vat",
44
+ "item_gross_worth",
45
+ "total_net_worth",
46
+ "total_vat",
47
+ "total_gross_worth",
48
+ ]
49
+
50
+ groups = (
51
+ [
52
+ "",
53
+ "items_row1",
54
+ "items_row2",
55
+ "items_row3",
56
+ "items_row4",
57
+ "items_row5",
58
+ "items_row6",
59
+ "items_row7",
60
+ "items_row8",
61
+ "items_row9",
62
+ "items_row10",
63
+ "summary",
64
+ ],
65
+ )
66
+
67
+ subgroups = ["", "subitem_row1", "subitem_row2", "subitem_row3"]
68
+
69
+ selected_field = "Selected Field: "
70
+ save_text = "Save"
71
+ saved_text = "Saved!"
72
+
73
+ subheader_1 = "Select"
74
+ subheader_2 = "Upload"
75
+ annotation_text = "Annotation"
76
+ no_annotation_file = "No annotation file selected"
77
+ no_annotation_mapping = "Please annotate the document. Uncheck 'Assign Labels' and draw new annotations"
78
+
79
+ download_text = "Download"
80
+ download_hint = "Download the annotated structure in JSON format"
81
+
82
+ annotation_selection_help = "Select an annotation file to load"
83
+ upload_help = "Upload a file to annotate"
84
+ upload_button_text = "Upload"
85
+ upload_button_text_desc = "Choose a file"
86
+
87
+ assign_labels_text = "Assign Labels"
88
+ assign_labels_help = "Check to enable editing of labels and values"
89
+
90
+ export_labels_text = "Export Labels"
91
+ export_labels_help = "Create key-value pairs for the labels in JSON format"
92
+ done_text = "Done"
93
+
94
+ grouping_id = "ID"
95
+ grouping_value = "Value"
96
+
97
+ completed_text = "Completed"
98
+ completed_help = "Check to mark the annotation as completed"
99
+
100
+ error_text = "Value is too long. Please shorten it."
101
+ selection_must_be_continuous = "Please select continuous rows"
102
+
103
+ def view(self, model, ui_width, device_type, device_width):
104
+ with open(model.labels_file, "r") as f:
105
+ labels_json = json.load(f)
106
+
107
+ labels_list = labels_json["labels"]
108
+ labels = [""]
109
+ for label in labels_list:
110
+ labels.append(label["name"])
111
+ model.labels = labels
112
+
113
+ with open(model.groups_file, "r") as f:
114
+ groups_json = json.load(f)
115
+
116
+ groups_list = groups_json["groups"]
117
+ groups = [""]
118
+ for group in groups_list:
119
+ groups.append(group["name"])
120
+ model.groups = groups
121
+
122
+ with open(model.subgroup_file, "r") as f:
123
+ subgroup_json = json.load(f)
124
+
125
+ subgroups_list = subgroup_json["subgroups"]
126
+ subgroups = [""]
127
+ for subgroup in subgroups_list:
128
+ subgroups.append(subgroup["name"])
129
+ model.subgroups = subgroups
130
+
131
+ with st.sidebar:
132
+ st.markdown("---")
133
+ st.subheader(model.subheader_1)
134
+
135
+ placeholder_upload = st.empty()
136
+
137
+ file_names = self.get_existing_file_names("docs/images/")
138
+
139
+ if "annotation_index" not in st.session_state:
140
+ st.session_state["annotation_index"] = 0
141
+ annotation_index = 0
142
+ else:
143
+ annotation_index = st.session_state["annotation_index"]
144
+
145
+ annotation_selection = placeholder_upload.selectbox(
146
+ model.annotation_text,
147
+ file_names,
148
+ index=annotation_index,
149
+ help=model.annotation_selection_help,
150
+ )
151
+
152
+ annotation_index = self.get_annotation_index(
153
+ annotation_selection, file_names
154
+ )
155
+
156
+ file_extension = self.get_file_extension(
157
+ annotation_selection, "docs/images/"
158
+ )
159
+
160
+ model.img_file = f"docs/images/{annotation_selection}" + file_extension
161
+ model.rects_file = f"docs/json/{annotation_selection}.json"
162
+
163
+ completed_check = st.empty()
164
+
165
+ btn = st.button(model.export_labels_text)
166
+ if btn:
167
+ self.export_labels(model)
168
+ st.write(model.done_text)
169
+
170
+ st.subheader(model.subheader_2)
171
+
172
+ with st.form("upload-form", clear_on_submit=True):
173
+ uploaded_file = st.file_uploader(
174
+ model.upload_button_text_desc,
175
+ accept_multiple_files=False,
176
+ type=["png", "jpg", "jpeg"],
177
+ help=model.upload_help,
178
+ )
179
+ submitted = st.form_submit_button(model.upload_button_text)
180
+
181
+ if submitted and uploaded_file is not None:
182
+ ret = self.upload_file(uploaded_file)
183
+
184
+ if ret is not False:
185
+ file_names = self.get_existing_file_names("docs/images/")
186
+
187
+ annotation_index = self.get_annotation_index(
188
+ annotation_selection, file_names
189
+ )
190
+ annotation_selection = placeholder_upload.selectbox(
191
+ model.annotation_text,
192
+ file_names,
193
+ index=annotation_index,
194
+ help=model.annotation_selection_help,
195
+ )
196
+ st.session_state["annotation_index"] = annotation_index
197
+
198
+ # st.title(model.pageTitle + " - " + annotation_selection)
199
+
200
+ if model.img_file is None:
201
+ st.caption(model.no_annotation_file)
202
+ return
203
+
204
+ saved_state = self.fetch_annotations(model.rects_file)
205
+
206
+ # annotation file has been changed
207
+ if annotation_index != st.session_state["annotation_index"]:
208
+ annotation_v = saved_state["meta"]["version"]
209
+ if annotation_v == "v0.1":
210
+ st.session_state["annotation_done"] = False
211
+ else:
212
+ st.session_state["annotation_done"] = True
213
+ # store the annotation file index
214
+ st.session_state["annotation_index"] = annotation_index
215
+
216
+ # first load
217
+ if "annotation_done" not in st.session_state:
218
+ annotation_v = saved_state["meta"]["version"]
219
+ if annotation_v == "v0.1":
220
+ st.session_state["annotation_done"] = False
221
+ else:
222
+ st.session_state["annotation_done"] = True
223
+
224
+ with completed_check:
225
+ annotation_done = st.checkbox(
226
+ model.completed_text, help=model.completed_help, key="annotation_done"
227
+ )
228
+ if annotation_done:
229
+ saved_state["meta"]["version"] = "v1.0"
230
+ else:
231
+ saved_state["meta"]["version"] = "v0.1"
232
+
233
+ with open(model.rects_file, "w") as f:
234
+ json.dump(saved_state, f, indent=2)
235
+ st.session_state[model.rects_file] = saved_state
236
+
237
+ assign_labels = st.checkbox(
238
+ model.assign_labels_text, True, help=model.assign_labels_help
239
+ )
240
+ mode = "transform" if assign_labels else "rect"
241
+
242
+ docImg = Image.open(model.img_file)
243
+
244
+ data_processor = DataProcessor()
245
+
246
+ with st.container():
247
+ doc_height = saved_state["meta"]["image_size"]["height"]
248
+ doc_width = saved_state["meta"]["image_size"]["width"]
249
+ canvas_width, number_of_columns = self.canvas_available_width(
250
+ ui_width, doc_width, device_type, device_width
251
+ )
252
+
253
+ if number_of_columns > 1:
254
+ col1, col2 = st.columns([number_of_columns, 10 - number_of_columns])
255
+ with col1:
256
+ result_rects = self.render_doc(
257
+ model,
258
+ docImg,
259
+ saved_state,
260
+ mode,
261
+ canvas_width,
262
+ doc_height,
263
+ doc_width,
264
+ )
265
+ with col2:
266
+ tab = st.radio(
267
+ "Select",
268
+ ["Mapping", "Grouping", "Ordering"],
269
+ horizontal=True,
270
+ label_visibility="collapsed",
271
+ )
272
+ if tab == "Mapping":
273
+ self.render_form(
274
+ model, result_rects, data_processor, annotation_selection
275
+ )
276
+ elif tab == "Grouping":
277
+ self.group_annotations(model, result_rects)
278
+ elif tab == "Ordering":
279
+ self.order_annotations(
280
+ model,
281
+ model.labels,
282
+ model.groups,
283
+ model.subgroups,
284
+ result_rects,
285
+ )
286
+ else:
287
+ result_rects = self.render_doc(
288
+ model,
289
+ docImg,
290
+ saved_state,
291
+ mode,
292
+ canvas_width,
293
+ doc_height,
294
+ doc_width,
295
+ )
296
+ tab = st.radio(
297
+ "Select",
298
+ ["Mapping", "Grouping"],
299
+ horizontal=True,
300
+ label_visibility="collapsed",
301
+ )
302
+ if tab == "Mapping":
303
+ self.render_form(
304
+ model, result_rects, data_processor, annotation_selection
305
+ )
306
+ else:
307
+ self.group_annotations(model, result_rects)
308
+
309
+ def render_doc(
310
+ self, model, docImg, saved_state, mode, canvas_width, doc_height, doc_width
311
+ ):
312
+ with st.container():
313
+ height = 1296
314
+ width = 864
315
+
316
+ result_rects = st_sparrow_labeling(
317
+ fill_color="rgba(0, 151, 255, 0.3)",
318
+ stroke_width=2,
319
+ stroke_color="rgba(0, 50, 255, 0.7)",
320
+ background_image=docImg,
321
+ initial_rects=saved_state,
322
+ height=height,
323
+ width=width,
324
+ drawing_mode=mode,
325
+ display_toolbar=True,
326
+ update_streamlit=True,
327
+ canvas_width=canvas_width,
328
+ doc_height=doc_height,
329
+ doc_width=doc_width,
330
+ image_rescale=True,
331
+ key="doc_annotation" + model.img_file,
332
+ )
333
+
334
+ st.caption(model.text_caption_1)
335
+ st.caption(model.text_caption_2)
336
+
337
+ return result_rects
338
+
339
+ def render_form(self, model, result_rects, data_processor, annotation_selection):
340
+ with st.container():
341
+ if result_rects is not None:
342
+ with st.form(key="fields_form"):
343
+ toolbar = st.empty()
344
+
345
+ self.render_form_view(
346
+ result_rects.rects_data["words"],
347
+ model.labels,
348
+ result_rects,
349
+ data_processor,
350
+ )
351
+
352
+ with toolbar:
353
+ submit = st.form_submit_button(model.save_text, type="primary")
354
+ if submit:
355
+ for word in result_rects.rects_data["words"]:
356
+ if len(word["value"]) > 1000:
357
+ st.error(model.error_text)
358
+ return
359
+
360
+ with open(model.rects_file, "w") as f:
361
+ json.dump(result_rects.rects_data, f, indent=2)
362
+ st.session_state[model.rects_file] = result_rects.rects_data
363
+ # st.write(model.saved_text)
364
+ st.experimental_rerun()
365
+
366
+ if len(result_rects.rects_data["words"]) == 0:
367
+ st.caption(model.no_annotation_mapping)
368
+ return
369
+ else:
370
+ with open(model.rects_file, "rb") as file:
371
+ st.download_button(
372
+ label=model.download_text,
373
+ data=file,
374
+ file_name=annotation_selection + ".json",
375
+ mime="application/json",
376
+ help=model.download_hint,
377
+ )
378
+
379
+ def render_form_view(self, words, labels, result_rects, data_processor):
380
+ data = []
381
+ for i, rect in enumerate(words):
382
+ group, label = (
383
+ rect["label"].split(":", 1)
384
+ if ":" in rect["label"]
385
+ else (None, rect["label"])
386
+ )
387
+ data.append({"id": i, "value": rect["value"], "label": label})
388
+ df = pd.DataFrame(data)
389
+
390
+ formatter = {
391
+ "id": ("ID", {**PINLEFT, "hide": True}),
392
+ "value": ("Value", {**PINLEFT, "editable": True}),
393
+ "label": (
394
+ "Label",
395
+ {
396
+ **PINLEFT,
397
+ "width": 80,
398
+ "editable": True,
399
+ "cellEditor": "agSelectCellEditor",
400
+ "cellEditorParams": {"values": labels},
401
+ },
402
+ ),
403
+ }
404
+
405
+ go = {
406
+ "rowClassRules": {
407
+ "row-selected": "data.id === " + str(result_rects.current_rect_index)
408
+ }
409
+ }
410
+
411
+ green_light = "#abf7b1"
412
+ css = {".row-selected": {"background-color": f"{green_light} !important"}}
413
+
414
+ response = agstyler.draw_grid(
415
+ df, formatter=formatter, fit_columns=True, grid_options=go, css=css
416
+ )
417
+
418
+ data = response["data"].values.tolist()
419
+
420
+ for i, rect in enumerate(words):
421
+ value = data[i][1]
422
+ label = data[i][2]
423
+ data_processor.update_rect_data(result_rects.rects_data, i, value, label)
424
+
425
+ def canvas_available_width(self, ui_width, doc_width, device_type, device_width):
426
+ CANVAS_WIDTH_SM = 37
427
+ CANVAS_WIDTH_MD = 55
428
+ CANVAS_WIDTH_LG = 60
429
+ doc_width_pct = (doc_width * 100) / ui_width
430
+ if doc_width_pct < 45:
431
+ canvas_width_pct = CANVAS_WIDTH_SM # 37
432
+ elif doc_width_pct < 55:
433
+ canvas_width_pct = CANVAS_WIDTH_MD
434
+ else:
435
+ canvas_width_pct = CANVAS_WIDTH_LG
436
+
437
+ if (
438
+ ui_width > 700
439
+ and canvas_width_pct == CANVAS_WIDTH_SM
440
+ and device_type == "desktop"
441
+ ):
442
+ return math.floor(canvas_width_pct * ui_width / 100), 4
443
+ elif (
444
+ ui_width > 700
445
+ and canvas_width_pct == CANVAS_WIDTH_MD
446
+ and device_type == "desktop"
447
+ ):
448
+ return math.floor(canvas_width_pct * ui_width / 100), 7
449
+ elif (
450
+ ui_width > 700
451
+ and canvas_width_pct == CANVAS_WIDTH_LG
452
+ and device_type == "desktop"
453
+ ):
454
+ return math.floor(canvas_width_pct * ui_width / 100), 1
455
+ else:
456
+ if device_type == "desktop":
457
+ ui_width = device_width - math.floor((device_width * 22) / 100)
458
+ elif device_type == "mobile":
459
+ ui_width = device_width - math.floor((device_width * 13) / 100)
460
+ return ui_width, 1
461
+
462
+ def fetch_annotations(self, rects_file):
463
+ for key in st.session_state:
464
+ if key.startswith("docs/json/") and key != rects_file:
465
+ del st.session_state[key]
466
+
467
+ if rects_file not in st.session_state:
468
+ if not os.path.exists(rects_file):
469
+ with open(rects_file, "w") as f:
470
+ saved_state = json.dump({}, f)
471
+ else:
472
+ with open(rects_file, "r") as f:
473
+ saved_state = json.load(f)
474
+
475
+ st.session_state[rects_file] = saved_state
476
+ else:
477
+ saved_state = st.session_state[rects_file]
478
+
479
+ return saved_state
480
+
481
+ def upload_file(self, uploaded_file):
482
+ if uploaded_file is not None:
483
+ if os.path.exists(os.path.join("docs/images/", uploaded_file.name)):
484
+ st.write("File already exists")
485
+ return False
486
+
487
+ if len(uploaded_file.name) > 100:
488
+ st.write("File name too long")
489
+ return False
490
+
491
+ with open(os.path.join("docs/images/", uploaded_file.name), "wb") as f:
492
+ f.write(uploaded_file.getbuffer())
493
+
494
+ img_file = Image.open(os.path.join("docs/images/", uploaded_file.name))
495
+
496
+ annotations_json = {
497
+ "meta": {
498
+ "version": "v0.1",
499
+ "split": "train",
500
+ "image_id": len(self.get_existing_file_names("docs/images/")),
501
+ "image_size": {"width": img_file.width, "height": img_file.height},
502
+ },
503
+ "words": [],
504
+ }
505
+
506
+ file_name = uploaded_file.name.split(".")[0]
507
+ with open(os.path.join("docs/json/", file_name + ".json"), "w") as f:
508
+ json.dump(annotations_json, f, indent=2)
509
+
510
+ st.success("File uploaded successfully")
511
+
512
+ def get_existing_file_names(self, dir_name):
513
+ # get ordered list of files without file extension, excluding hidden files
514
+ return natsorted(
515
+ [
516
+ os.path.splitext(f)[0]
517
+ for f in os.listdir(dir_name)
518
+ if not f.startswith(".")
519
+ ]
520
+ )
521
+
522
+ def get_file_extension(self, file_name, dir_name):
523
+ # get list of files, excluding hidden files
524
+ files = [f for f in os.listdir(dir_name) if not f.startswith(".")]
525
+ for f in files:
526
+ if file_name is not None and os.path.splitext(f)[0] == file_name:
527
+ return os.path.splitext(f)[1]
528
+
529
+ def get_annotation_index(self, file, files_list):
530
+ return files_list.index(file)
531
+
532
+ def group_annotations(self, model, result_rects):
533
+ with st.form(key="grouping_form"):
534
+ if result_rects is not None:
535
+ words = result_rects.rects_data["words"]
536
+ data = []
537
+ for i, rect in enumerate(words):
538
+ data.append({"id": i, "value": rect["value"]})
539
+ df = pd.DataFrame(data)
540
+
541
+ formatter = {
542
+ "id": ("ID", {**PINLEFT, "width": 50}),
543
+ "value": ("Value", PINLEFT),
544
+ }
545
+
546
+ toolbar = st.empty()
547
+
548
+ response = agstyler.draw_grid(
549
+ df,
550
+ formatter=formatter,
551
+ fit_columns=True,
552
+ selection="multiple",
553
+ use_checkbox="True",
554
+ pagination_size=40,
555
+ )
556
+
557
+ rows = response["selected_rows"]
558
+
559
+ with toolbar:
560
+ submit = st.form_submit_button(model.save_text, type="primary")
561
+ if submit and len(rows) > 0:
562
+ # check if there are gaps in the selected rows
563
+ if len(rows) > 1:
564
+ for i in range(len(rows) - 1):
565
+ if rows[i]["id"] + 1 != rows[i + 1]["id"]:
566
+ st.error(model.selection_must_be_continuous)
567
+ return
568
+
569
+ words = result_rects.rects_data["words"]
570
+ new_words_list = []
571
+ coords = []
572
+ for row in rows:
573
+ word_value = words[row["id"]]["value"]
574
+ rect = words[row["id"]]["rect"]
575
+ coords.append(rect)
576
+ new_words_list.append(word_value)
577
+ # convert array to string
578
+ new_word = " ".join(new_words_list)
579
+
580
+ # Get min x1 value from coords array
581
+ x1_min = min([coord["x1"] for coord in coords])
582
+ y1_min = min([coord["y1"] for coord in coords])
583
+ x2_max = max([coord["x2"] for coord in coords])
584
+ y2_max = max([coord["y2"] for coord in coords])
585
+
586
+ words[rows[0]["id"]]["value"] = new_word
587
+ words[rows[0]["id"]]["rect"] = {
588
+ "x1": x1_min,
589
+ "y1": y1_min,
590
+ "x2": x2_max,
591
+ "y2": y2_max,
592
+ }
593
+
594
+ # loop array in reverse order and remove selected entries
595
+ i = 0
596
+ for row in rows[::-1]:
597
+ if i == len(rows) - 1:
598
+ break
599
+ del words[row["id"]]
600
+ i += 1
601
+
602
+ result_rects.rects_data["words"] = words
603
+
604
+ with open(model.rects_file, "w") as f:
605
+ json.dump(result_rects.rects_data, f, indent=2)
606
+ st.session_state[model.rects_file] = result_rects.rects_data
607
+ st.experimental_rerun()
608
+
609
+ def order_annotations(self, model, labels, groups, subgroups, result_rects):
610
+ if result_rects is not None:
611
+ self.action_event = None
612
+ data = []
613
+ idx_list = [""]
614
+ words = result_rects.rects_data["words"]
615
+ for i, rect in enumerate(words):
616
+ if rect["label"] != "":
617
+ # split string into two variables, assign None to first variable if no split is found
618
+ # parts = (
619
+ # rect["label"].split(":", 2)
620
+ # if ":" in rect["label"]
621
+ # else (None, rect["label"])
622
+ # )
623
+ if ":" in rect["label"]:
624
+ parts = rect["label"].split(":", 2)
625
+ else:
626
+ parts = (None, rect["label"])
627
+ if parts:
628
+ group = parts[0]
629
+ label = parts[-1]
630
+ subgroup = parts[1] if len(parts) == 3 else None
631
+ data.append(
632
+ {
633
+ "id": i,
634
+ "value": rect["value"],
635
+ "label": label,
636
+ "group": group,
637
+ "subgroup": subgroup,
638
+ }
639
+ )
640
+ idx_list.append(i)
641
+ df = pd.DataFrame(data)
642
+
643
+ formatter = {
644
+ "id": ("ID", {**PINLEFT, "width": 50}),
645
+ "value": ("Value", {**PINLEFT}),
646
+ "label": (
647
+ "Label",
648
+ {
649
+ **PINLEFT,
650
+ "width": 80,
651
+ "editable": False,
652
+ "cellEditor": "agSelectCellEditor",
653
+ "cellEditorParams": {"values": labels},
654
+ },
655
+ ),
656
+ "group": (
657
+ "Group",
658
+ {
659
+ **PINLEFT,
660
+ "width": 80,
661
+ "editable": True,
662
+ "cellEditor": "agSelectCellEditor",
663
+ "cellEditorParams": {"values": groups},
664
+ },
665
+ ),
666
+ "subgroup": (
667
+ "Sub Group",
668
+ {
669
+ **PINLEFT,
670
+ "width": 80,
671
+ "editable": True,
672
+ "cellEditor": "agSelectCellEditor",
673
+ "cellEditorParams": {"values": subgroups},
674
+ },
675
+ ),
676
+ }
677
+
678
+ go = {
679
+ "rowClassRules": {
680
+ "row-selected": "data.id === "
681
+ + str(result_rects.current_rect_index)
682
+ }
683
+ }
684
+
685
+ green_light = "#abf7b1"
686
+ css = {".row-selected": {"background-color": f"{green_light} !important"}}
687
+
688
+ idx_option = st.selectbox("Select row to move into", idx_list)
689
+
690
+ def run_component(props):
691
+ value = component_toolbar_main(key="toolbar_main", **props)
692
+ return value
693
+
694
+ def handle_event(value):
695
+ if value is not None:
696
+ if "action_timestamp" not in st.session_state:
697
+ self.action_event = value["action"]
698
+ st.session_state["action_timestamp"] = value["timestamp"]
699
+ else:
700
+ if st.session_state["action_timestamp"] != value["timestamp"]:
701
+ self.action_event = value["action"]
702
+ st.session_state["action_timestamp"] = value["timestamp"]
703
+ else:
704
+ self.action_event = None
705
+
706
+ props = {
707
+ "buttons": {
708
+ "up": {"disabled": False, "rendered": ""},
709
+ "down": {"disabled": False, "rendered": ""},
710
+ "save": {
711
+ "disabled": False,
712
+ "rendered": "",
713
+ # 'rendered': 'none',
714
+ },
715
+ }
716
+ }
717
+
718
+ handle_event(run_component(props))
719
+
720
+ response = agstyler.draw_grid(
721
+ df, formatter=formatter, fit_columns=True, grid_options=go, css=css
722
+ )
723
+
724
+ rows = response["selected_rows"]
725
+ if len(rows) == 0 and result_rects.current_rect_index > -1:
726
+ for i, row in enumerate(data):
727
+ if row["id"] == result_rects.current_rect_index:
728
+ rows = [
729
+ {
730
+ "_selectedRowNodeInfo": {"nodeRowIndex": i},
731
+ "id": row["id"],
732
+ }
733
+ ]
734
+ break
735
+
736
+ if str(self.action_event) == "up":
737
+ if len(rows) > 0:
738
+ idx = rows[0]["_selectedRowNodeInfo"]["nodeRowIndex"]
739
+ if idx > 0:
740
+ row_id = rows[0]["id"]
741
+ if row_id == idx_option:
742
+ return
743
+ # swap row upwards in the array
744
+ if idx_option == "":
745
+ words[row_id], words[row_id - 1] = (
746
+ words[row_id - 1],
747
+ words[row_id],
748
+ )
749
+ else:
750
+ for i in range(1000):
751
+ words[row_id], words[row_id - 1] = (
752
+ words[row_id - 1],
753
+ words[row_id],
754
+ )
755
+ row_id -= 1
756
+ if row_id == idx_option:
757
+ break
758
+
759
+ result_rects.rects_data["words"] = words
760
+
761
+ with open(model.rects_file, "w") as f:
762
+ json.dump(result_rects.rects_data, f, indent=2)
763
+ st.session_state[model.rects_file] = result_rects.rects_data
764
+ st.experimental_rerun()
765
+ elif str(self.action_event) == "down":
766
+ if len(rows) > 0:
767
+ idx = rows[0]["_selectedRowNodeInfo"]["nodeRowIndex"]
768
+ if idx < len(df) - 1:
769
+ row_id = rows[0]["id"]
770
+ if row_id == idx_option:
771
+ return
772
+ # swap row downwards in the array
773
+ if idx_option == "":
774
+ words[row_id], words[row_id + 1] = (
775
+ words[row_id + 1],
776
+ words[row_id],
777
+ )
778
+ else:
779
+ for i in range(1000):
780
+ words[row_id], words[row_id + 1] = (
781
+ words[row_id + 1],
782
+ words[row_id],
783
+ )
784
+ row_id += 1
785
+ if row_id == idx_option:
786
+ break
787
+
788
+ result_rects.rects_data["words"] = words
789
+
790
+ with open(model.rects_file, "w") as f:
791
+ json.dump(result_rects.rects_data, f, indent=2)
792
+ st.session_state[model.rects_file] = result_rects.rects_data
793
+ st.experimental_rerun()
794
+ elif str(self.action_event) == "save":
795
+ data = response["data"].values.tolist()
796
+ for elem in data:
797
+ if elem[3] != "None":
798
+ idx = elem[0]
799
+ group = elem[3]
800
+ subgroup = elem[4]
801
+
802
+ if subgroup and subgroup != "None":
803
+ words[idx]["label"] = f"{group}:{subgroup}:{elem[2]}"
804
+ else:
805
+ words[idx]["label"] = f"{group}:{elem[2]}"
806
+ result_rects.rects_data["words"] = words
807
+
808
+ with open(model.rects_file, "w") as f:
809
+ json.dump(result_rects.rects_data, f, indent=2)
810
+ st.session_state[model.rects_file] = result_rects.rects_data
811
+ st.experimental_rerun()
812
+
813
+ def export_labels(self, model):
814
+ path_from = os.path.join("docs/json/")
815
+ path_to = os.path.join("docs/json/key/")
816
+
817
+ files = [f for f in os.listdir(path_from) if not f.startswith(".")]
818
+ for file in files:
819
+ path = os.path.join(path_from, file)
820
+ if os.path.isfile(path):
821
+ with open(path, "r") as f:
822
+ data = json.load(f)
823
+ words = data["words"]
824
+
825
+ keys = {}
826
+ row_keys = {}
827
+ subrow_keys = {}
828
+
829
+ for word in words:
830
+ if word["label"] != "":
831
+ if ":" in word["label"]:
832
+ group, label = word["label"].split(":", 1)
833
+ if "row" not in group:
834
+ if group not in keys:
835
+ keys[group] = {}
836
+ keys[group][label] = word["value"]
837
+ else:
838
+ if "items" not in keys:
839
+ keys["items"] = []
840
+
841
+ if group not in row_keys:
842
+ row_keys[group] = {}
843
+
844
+ if "subitem" in label:
845
+ subgroup, sublabel = label.split(":", 1)
846
+ if "subitems" not in row_keys[group]:
847
+ row_keys[group]["subitems"] = []
848
+
849
+ if subgroup not in subrow_keys:
850
+ subrow_keys[subgroup] = {}
851
+
852
+ subrow_keys[subgroup][sublabel] = word["value"]
853
+
854
+ if (
855
+ subrow_keys[subgroup]
856
+ not in row_keys[group]["subitems"]
857
+ ):
858
+ row_keys[group]["subitems"].append(
859
+ subrow_keys[subgroup]
860
+ )
861
+
862
+ row_keys[group][label] = word["value"]
863
+
864
+ keys_to_delete = [
865
+ key
866
+ for key in row_keys[group].keys()
867
+ if "subitem_row" in key
868
+ ]
869
+
870
+ for key in keys_to_delete:
871
+ if "subitem_row" in key:
872
+ del row_keys[group][key]
873
+ else:
874
+ keys[word["label"]] = word["value"]
875
+
876
+ if row_keys != {}:
877
+ for key in row_keys:
878
+ keys["items"].append(row_keys[key])
879
+
880
+ if keys != {}:
881
+ path = os.path.join(path_to, file)
882
+ with open(path, "w") as f:
883
+ json.dump(keys, f, indent=2)
views/data_inference.py ADDED
@@ -0,0 +1,219 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+ import os
3
+ import time
4
+ from PIL import Image
5
+ import math
6
+ from streamlit_sparrow_labeling import st_sparrow_labeling
7
+ import requests
8
+ from config import settings
9
+ import json
10
+
11
+
12
+ class DataInference:
13
+ class Model:
14
+ # pageTitle = "Data Inference"
15
+ subheader_2 = "Upload"
16
+ initial_msg = "Please upload a file for inference"
17
+
18
+ upload_help = "Upload a file to extract data from it"
19
+ upload_button_text = "Upload"
20
+ upload_button_text_desc = "Choose a file"
21
+
22
+ extract_data = "Extract Data"
23
+
24
+ model_in_use = "donut"
25
+
26
+ img_file = None
27
+
28
+ def set_image_file(self, img_file):
29
+ st.session_state['img_file'] = img_file
30
+
31
+ def get_image_file(self):
32
+ if 'img_file' not in st.session_state:
33
+ return None
34
+ return st.session_state['img_file']
35
+
36
+ data_result = None
37
+
38
+ def set_data_result(self, data_result):
39
+ st.session_state['data_result'] = data_result
40
+
41
+ def get_data_result(self):
42
+ if 'data_result' not in st.session_state:
43
+ return None
44
+ return st.session_state['data_result']
45
+
46
+ def view(self, model, ui_width, device_type, device_width):
47
+ # st.title(model.pageTitle)
48
+
49
+ with st.sidebar:
50
+ st.markdown("---")
51
+ st.subheader(model.subheader_2)
52
+
53
+ with st.form("upload-form", clear_on_submit=True):
54
+ uploaded_file = st.file_uploader(model.upload_button_text_desc, accept_multiple_files=False,
55
+ type=['png', 'jpg', 'jpeg'],
56
+ help=model.upload_help)
57
+ submitted = st.form_submit_button(model.upload_button_text)
58
+
59
+ if submitted and uploaded_file is not None:
60
+ ret = self.upload_file(uploaded_file)
61
+
62
+ if ret is not False:
63
+ model.set_image_file(ret)
64
+ model.set_data_result(None)
65
+
66
+ if model.get_image_file() is not None:
67
+ doc_img = Image.open(model.get_image_file())
68
+ doc_height = doc_img.height
69
+ doc_width = doc_img.width
70
+
71
+ canvas_width, number_of_columns = self.canvas_available_width(ui_width, doc_width, device_type,
72
+ device_width)
73
+
74
+ if number_of_columns > 1:
75
+ col1, col2 = st.columns([number_of_columns, 10 - number_of_columns])
76
+ with col1:
77
+ self.render_doc(model, doc_img, canvas_width, doc_height, doc_width)
78
+ with col2:
79
+ self.render_results(model)
80
+ else:
81
+ self.render_doc(model, doc_img, canvas_width, doc_height, doc_width)
82
+ self.render_results(model)
83
+ else:
84
+ st.title(model.initial_msg)
85
+
86
+ def upload_file(self, uploaded_file):
87
+ timestamp = str(time.time())
88
+ timestamp = timestamp.replace(".", "")
89
+
90
+ file_name, file_extension = os.path.splitext(uploaded_file.name)
91
+ uploaded_file.name = file_name + "_" + timestamp + file_extension
92
+
93
+ if os.path.exists(os.path.join("docs/inference/", uploaded_file.name)):
94
+ st.write("File already exists")
95
+ return False
96
+
97
+ if len(uploaded_file.name) > 500:
98
+ st.write("File name too long")
99
+ return False
100
+
101
+ with open(os.path.join("docs/inference/", uploaded_file.name), "wb") as f:
102
+ f.write(uploaded_file.getbuffer())
103
+
104
+ st.success("File uploaded successfully")
105
+
106
+ return os.path.join("docs/inference/", uploaded_file.name)
107
+
108
+ def canvas_available_width(self, ui_width, doc_width, device_type, device_width):
109
+ doc_width_pct = (doc_width * 100) / ui_width
110
+ if doc_width_pct < 45:
111
+ canvas_width_pct = 37
112
+ elif doc_width_pct < 55:
113
+ canvas_width_pct = 49
114
+ else:
115
+ canvas_width_pct = 60
116
+
117
+ if ui_width > 700 and canvas_width_pct == 37 and device_type == "desktop":
118
+ return math.floor(canvas_width_pct * ui_width / 100), 4
119
+ elif ui_width > 700 and canvas_width_pct == 49 and device_type == "desktop":
120
+ return math.floor(canvas_width_pct * ui_width / 100), 5
121
+ elif ui_width > 700 and canvas_width_pct == 60 and device_type == "desktop":
122
+ return math.floor(canvas_width_pct * ui_width / 100), 6
123
+ else:
124
+ if device_type == "desktop":
125
+ ui_width = device_width - math.floor((device_width * 22) / 100)
126
+ elif device_type == "mobile":
127
+ ui_width = device_width - math.floor((device_width * 13) / 100)
128
+ return ui_width, 1
129
+
130
+ def render_doc(self, model, doc_img, canvas_width, doc_height, doc_width):
131
+ height = 1296
132
+ width = 864
133
+
134
+ annotations_json = {
135
+ "meta": {
136
+ "version": "v0.1",
137
+ "split": "train",
138
+ "image_id": 0,
139
+ "image_size": {
140
+ "width": doc_width,
141
+ "height": doc_height
142
+ }
143
+ },
144
+ "words": []
145
+ }
146
+
147
+ st_sparrow_labeling(
148
+ fill_color="rgba(0, 151, 255, 0.3)",
149
+ stroke_width=2,
150
+ stroke_color="rgba(0, 50, 255, 0.7)",
151
+ background_image=doc_img,
152
+ initial_rects=annotations_json,
153
+ height=height,
154
+ width=width,
155
+ drawing_mode="transform",
156
+ display_toolbar=False,
157
+ update_streamlit=False,
158
+ canvas_width=canvas_width,
159
+ doc_height=doc_height,
160
+ doc_width=doc_width,
161
+ image_rescale=True,
162
+ key="doc_annotation" + model.get_image_file()
163
+ )
164
+
165
+ def render_results(self, model):
166
+ with st.form(key="results_form"):
167
+ button_placeholder = st.empty()
168
+
169
+ submit = button_placeholder.form_submit_button(model.extract_data, type="primary")
170
+ if 'inference_error' in st.session_state:
171
+ st.error(st.session_state.inference_error)
172
+ del st.session_state.inference_error
173
+
174
+ if submit:
175
+ button_placeholder.empty()
176
+
177
+ api_url = "https://katanaml-org-sparrow-ml.hf.space/api-inference/v1/sparrow-ml/inference"
178
+ file_path = model.get_image_file()
179
+
180
+ with open(file_path, "rb") as file:
181
+ model_in_use = model.model_in_use
182
+ sparrow_key = settings.sparrow_key
183
+
184
+ # Prepare the payload
185
+ files = {
186
+ 'file': (file.name, file, 'image/jpeg')
187
+ }
188
+
189
+ data = {
190
+ 'image_url': '',
191
+ 'model_in_use': model_in_use,
192
+ 'sparrow_key': sparrow_key
193
+ }
194
+
195
+ with st.spinner("Extracting data from document..."):
196
+ response = requests.post(api_url, data=data, files=files, timeout=180)
197
+ if response.status_code != 200:
198
+ print('Request failed with status code:', response.status_code)
199
+ print('Response:', response.text)
200
+
201
+ st.session_state["inference_error"] = "Error extracting data from document"
202
+ st.experimental_rerun()
203
+
204
+ model.set_data_result(response.text)
205
+
206
+ # Display JSON data in Streamlit
207
+ st.markdown("---")
208
+ st.json(response.text)
209
+
210
+ # replace file extension to json
211
+ file_path = file_path.replace(".jpg", ".json")
212
+ with open(file_path, "w") as f:
213
+ json.dump(response.text, f, indent=2)
214
+
215
+ st.experimental_rerun()
216
+ else:
217
+ if model.get_data_result() is not None:
218
+ st.markdown("---")
219
+ st.json(model.get_data_result())
views/data_review.py ADDED
@@ -0,0 +1,165 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+ from natsort import natsorted
3
+ import os
4
+ from PIL import Image
5
+ import math
6
+ from streamlit_sparrow_labeling import st_sparrow_labeling
7
+ import json
8
+
9
+
10
+ class DataReview:
11
+ class Model:
12
+ # pageTitle = "Data Review"
13
+ subheader_2 = "Select"
14
+ subheader_3 = "Result"
15
+ selection_text = "File to review"
16
+ initial_msg = "Please select a file to review"
17
+
18
+ img_file = None
19
+
20
+ def set_image_file(self, img_file):
21
+ st.session_state['img_file_review'] = img_file
22
+
23
+ def get_image_file(self):
24
+ if 'img_file_review' not in st.session_state:
25
+ return None
26
+ return st.session_state['img_file_review']
27
+
28
+ json_file = None
29
+
30
+ def set_json_file(self, json_file):
31
+ st.session_state['json_file_review'] = json_file
32
+
33
+ def get_json_file(self):
34
+ if 'json_file_review' not in st.session_state:
35
+ return None
36
+ return st.session_state['json_file_review']
37
+
38
+ def view(self, model, ui_width, device_type, device_width):
39
+ # st.title(model.pageTitle)
40
+
41
+ with st.sidebar:
42
+ st.markdown("---")
43
+ st.subheader(model.subheader_2)
44
+
45
+ # get list of files in inference directory
46
+ processed_file_names = self.get_processed_file_names('docs/inference/')
47
+
48
+ if 'selection_index' not in st.session_state:
49
+ st.session_state['selection_index'] = 0
50
+ selection_index = 0
51
+ else:
52
+ selection_index = st.session_state['selection_index']
53
+
54
+ selection = st.selectbox(model.selection_text, processed_file_names, index=selection_index)
55
+
56
+ selection_index = self.get_selection_index(selection, processed_file_names)
57
+ st.session_state['selection_index'] = selection_index
58
+
59
+ img_file = "docs/inference/" + selection + ".jpg"
60
+ json_file = "docs/inference/" + selection + ".json"
61
+
62
+ model.set_image_file(img_file)
63
+ model.set_json_file(json_file)
64
+
65
+ if model.get_image_file() is not None:
66
+ doc_img = Image.open(model.get_image_file())
67
+ doc_height = doc_img.height
68
+ doc_width = doc_img.width
69
+
70
+ canvas_width, number_of_columns = self.canvas_available_width(ui_width, doc_width, device_type,
71
+ device_width)
72
+
73
+ if number_of_columns > 1:
74
+ col1, col2 = st.columns([number_of_columns, 10 - number_of_columns])
75
+ with col1:
76
+ pass
77
+ self.render_doc(model, doc_img, canvas_width, doc_height, doc_width)
78
+ with col2:
79
+ pass
80
+ self.render_results(model)
81
+ else:
82
+ pass
83
+ self.render_doc(model, doc_img, canvas_width, doc_height, doc_width)
84
+ self.render_results(model)
85
+ else:
86
+ st.title(model.initial_msg)
87
+
88
+
89
+ def get_processed_file_names(self, dir_name):
90
+ # get ordered list of files without file extension, excluding hidden files, with JSON extension only
91
+ file_names = [os.path.splitext(f)[0] for f in os.listdir(dir_name) if
92
+ os.path.isfile(os.path.join(dir_name, f)) and not f.startswith('.') and f.endswith('.json')]
93
+ file_names = natsorted(file_names)
94
+ return file_names
95
+
96
+ def get_selection_index(self, file, files_list):
97
+ return files_list.index(file)
98
+
99
+ def canvas_available_width(self, ui_width, doc_width, device_type, device_width):
100
+ doc_width_pct = (doc_width * 100) / ui_width
101
+ if doc_width_pct < 45:
102
+ canvas_width_pct = 37
103
+ elif doc_width_pct < 55:
104
+ canvas_width_pct = 49
105
+ else:
106
+ canvas_width_pct = 60
107
+
108
+ if ui_width > 700 and canvas_width_pct == 37 and device_type == "desktop":
109
+ return math.floor(canvas_width_pct * ui_width / 100), 4
110
+ elif ui_width > 700 and canvas_width_pct == 49 and device_type == "desktop":
111
+ return math.floor(canvas_width_pct * ui_width / 100), 5
112
+ elif ui_width > 700 and canvas_width_pct == 60 and device_type == "desktop":
113
+ return math.floor(canvas_width_pct * ui_width / 100), 6
114
+ else:
115
+ if device_type == "desktop":
116
+ ui_width = device_width - math.floor((device_width * 22) / 100)
117
+ elif device_type == "mobile":
118
+ ui_width = device_width - math.floor((device_width * 13) / 100)
119
+ return ui_width, 1
120
+
121
+
122
+ def render_doc(self, model, doc_img, canvas_width, doc_height, doc_width):
123
+ height = 1296
124
+ width = 864
125
+
126
+ annotations_json = {
127
+ "meta": {
128
+ "version": "v0.1",
129
+ "split": "train",
130
+ "image_id": 0,
131
+ "image_size": {
132
+ "width": doc_width,
133
+ "height": doc_height
134
+ }
135
+ },
136
+ "words": []
137
+ }
138
+
139
+ st_sparrow_labeling(
140
+ fill_color="rgba(0, 151, 255, 0.3)",
141
+ stroke_width=2,
142
+ stroke_color="rgba(0, 50, 255, 0.7)",
143
+ background_image=doc_img,
144
+ initial_rects=annotations_json,
145
+ height=height,
146
+ width=width,
147
+ drawing_mode="transform",
148
+ display_toolbar=False,
149
+ update_streamlit=False,
150
+ canvas_width=canvas_width,
151
+ doc_height=doc_height,
152
+ doc_width=doc_width,
153
+ image_rescale=True,
154
+ key="doc_annotation" + model.get_image_file()
155
+ )
156
+
157
+ def render_results(self, model):
158
+ json_file = model.get_json_file()
159
+ if json_file is not None:
160
+ with open(json_file) as f:
161
+ data_json = json.load(f)
162
+ st.subheader(model.subheader_3)
163
+ st.markdown("---")
164
+ st.json(data_json)
165
+ st.markdown("---")
views/model_training.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+
3
+
4
+ class ModelTraining:
5
+ class Model:
6
+ pageTitle = "Model Training"
7
+
8
+ def view(self, model):
9
+ st.title(model.pageTitle)
views/model_tuning.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+
3
+
4
+ class ModelTuning:
5
+ class Model:
6
+ pageTitle = "Model Tuning"
7
+
8
+ def view(self, model):
9
+ st.title(model.pageTitle)
views/setup.py ADDED
@@ -0,0 +1,379 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+ import json
3
+ import pandas as pd
4
+ from tools import agstyler
5
+ from tools.agstyler import PINLEFT
6
+ from toolbar import component_toolbar_buttons
7
+
8
+
9
+ class Setup:
10
+ class Model:
11
+ header1 = "Labels"
12
+ header2 = "Groups"
13
+ header3 = "Sub Group"
14
+ labels_file = "docs/labels.json"
15
+ groups_file = "docs/groups.json"
16
+ subgroups_file = "docs/subgroups.json"
17
+
18
+ def view(self, model):
19
+ tab = st.radio(
20
+ "Select",
21
+ ["Labels", "Groups", "Sub Group"],
22
+ horizontal=True,
23
+ label_visibility="collapsed",
24
+ )
25
+
26
+ if tab == "Labels":
27
+ st.title(model.header1)
28
+ self.setup_labels(model)
29
+ elif tab == "Groups":
30
+ st.title(model.header2)
31
+ self.setup_groups(model)
32
+ elif tab == "Sub Group":
33
+ st.title(model.header3)
34
+ self.setup_subgroups(model)
35
+
36
+ def setup_labels(self, model):
37
+ self.action_event = False
38
+ if "action" not in st.session_state:
39
+ st.session_state["action"] = None
40
+
41
+ with open(model.labels_file, "r") as f:
42
+ labels_json = json.load(f)
43
+
44
+ labels = labels_json["labels"]
45
+
46
+ data = []
47
+ for label in labels:
48
+ data.append(
49
+ {
50
+ "id": label["id"],
51
+ "name": label["name"],
52
+ "description": label["description"],
53
+ }
54
+ )
55
+ self.df = pd.DataFrame(data)
56
+
57
+ formatter = {
58
+ "id": ("ID", {"hide": True}),
59
+ "name": ("Label", {**PINLEFT, "editable": True}),
60
+ "description": ("Description", {**PINLEFT, "editable": True}),
61
+ }
62
+
63
+ def run_component(props):
64
+ value = component_toolbar_buttons(key="toolbar_buttons_labels", **props)
65
+ return value
66
+
67
+ def handle_event(value):
68
+ if value is not None:
69
+ if "action_timestamp" not in st.session_state:
70
+ self.action_event = True
71
+ st.session_state["action_timestamp"] = value["timestamp"]
72
+ else:
73
+ if st.session_state["action_timestamp"] != value["timestamp"]:
74
+ self.action_event = True
75
+ st.session_state["action_timestamp"] = value["timestamp"]
76
+ else:
77
+ self.action_event = False
78
+
79
+ if value is not None and value["action"] == "create" and self.action_event:
80
+ if st.session_state["action"] != "delete":
81
+ max_id = self.df["id"].max()
82
+ self.df.loc[-1] = [max_id + 1, "", ""] # adding a row
83
+ self.df.index = self.df.index + 1 # shifting index
84
+ self.df.sort_index(inplace=True)
85
+ st.session_state["action"] = "create"
86
+ elif (
87
+ value is not None and value["action"] == "delete" and self.action_event
88
+ ):
89
+ if (
90
+ st.session_state["action"] != "delete"
91
+ and st.session_state["action"] != "create"
92
+ ):
93
+ rows = st.session_state["selected_rows"]
94
+ if len(rows) > 0:
95
+ idx = rows[0]["_selectedRowNodeInfo"]["nodeRowIndex"]
96
+ self.df.drop(self.df.index[idx], inplace=True)
97
+ self.df.reset_index(drop=True, inplace=True)
98
+ st.session_state["action"] = "delete"
99
+ elif value is not None and value["action"] == "save" and self.action_event:
100
+ st.session_state["action"] = "save"
101
+
102
+ props = {
103
+ "buttons": {
104
+ "create": False,
105
+ "delete": False,
106
+ "save": False,
107
+ }
108
+ }
109
+
110
+ handle_event(run_component(props))
111
+
112
+ if st.session_state["action"] == "save" and "response" in st.session_state:
113
+ if st.session_state["response"] is not None:
114
+ self.df = st.session_state["response"]
115
+ st.session_state["response"] = None
116
+
117
+ if st.session_state["action"] == "create" and "response" in st.session_state:
118
+ if st.session_state["response"] is not None:
119
+ self.df = st.session_state["response"]
120
+
121
+ if st.session_state["action"] == "delete" and "response" in st.session_state:
122
+ if st.session_state["response"] is not None:
123
+ self.df = st.session_state["response"]
124
+
125
+ response = agstyler.draw_grid(
126
+ self.df,
127
+ formatter=formatter,
128
+ fit_columns=True,
129
+ pagination_size=10,
130
+ selection="single",
131
+ use_checkbox=False,
132
+ )
133
+
134
+ rows = response["selected_rows"]
135
+ st.session_state["selected_rows"] = rows
136
+
137
+ if st.session_state["action"] == "create" and self.action_event:
138
+ st.session_state["response"] = response["data"]
139
+ elif st.session_state["action"] == "delete" and self.action_event:
140
+ st.session_state["response"] = response["data"]
141
+ elif st.session_state["action"] == "save" and self.action_event:
142
+ data = response["data"].values.tolist()
143
+ rows = []
144
+ for row in data:
145
+ rows.append({"id": row[0], "name": row[1], "description": row[2]})
146
+
147
+ labels_json["labels"] = rows
148
+ with open(model.labels_file, "w") as f:
149
+ json.dump(labels_json, f, indent=2)
150
+
151
+ def setup_groups(self, model):
152
+ self.action_event = False
153
+ if "action" not in st.session_state:
154
+ st.session_state["action"] = None
155
+
156
+ with open(model.groups_file, "r") as f:
157
+ groups_json = json.load(f)
158
+
159
+ groups = groups_json["groups"]
160
+
161
+ data = []
162
+ for group in groups:
163
+ data.append(
164
+ {
165
+ "id": group["id"],
166
+ "name": group["name"],
167
+ "description": group["description"],
168
+ }
169
+ )
170
+ self.df = pd.DataFrame(data)
171
+
172
+ formatter = {
173
+ "id": ("ID", {"hide": True}),
174
+ "name": ("Group", {**PINLEFT, "editable": True}),
175
+ "description": ("Description", {**PINLEFT, "editable": True}),
176
+ }
177
+
178
+ def run_component(props):
179
+ value = component_toolbar_buttons(key="toolbar_buttons_groups", **props)
180
+ return value
181
+
182
+ def handle_event(value):
183
+ if value is not None:
184
+ if "action_timestamp" not in st.session_state:
185
+ self.action_event = True
186
+ st.session_state["action_timestamp"] = value["timestamp"]
187
+ else:
188
+ if st.session_state["action_timestamp"] != value["timestamp"]:
189
+ self.action_event = True
190
+ st.session_state["action_timestamp"] = value["timestamp"]
191
+ else:
192
+ self.action_event = False
193
+
194
+ if value is not None and value["action"] == "create" and self.action_event:
195
+ if st.session_state["action"] != "delete":
196
+ max_id = self.df["id"].max()
197
+ self.df.loc[-1] = [max_id + 1, "", ""] # adding a row
198
+ self.df.index = self.df.index + 1 # shifting index
199
+ self.df.sort_index(inplace=True)
200
+ st.session_state["action"] = "create"
201
+ elif (
202
+ value is not None and value["action"] == "delete" and self.action_event
203
+ ):
204
+ if (
205
+ st.session_state["action"] != "delete"
206
+ and st.session_state["action"] != "create"
207
+ ):
208
+ rows = st.session_state["selected_rows"]
209
+ if len(rows) > 0:
210
+ idx = rows[0]["_selectedRowNodeInfo"]["nodeRowIndex"]
211
+ self.df.drop(self.df.index[idx], inplace=True)
212
+ self.df.reset_index(drop=True, inplace=True)
213
+ st.session_state["action"] = "delete"
214
+ elif value is not None and value["action"] == "save" and self.action_event:
215
+ st.session_state["action"] = "save"
216
+
217
+ props = {
218
+ "buttons": {
219
+ "create": False,
220
+ "delete": False,
221
+ "save": False,
222
+ }
223
+ }
224
+
225
+ handle_event(run_component(props))
226
+
227
+ if st.session_state["action"] == "save" and "response" in st.session_state:
228
+ if st.session_state["response"] is not None:
229
+ self.df = st.session_state["response"]
230
+ st.session_state["response"] = None
231
+
232
+ if st.session_state["action"] == "create" and "response" in st.session_state:
233
+ if st.session_state["response"] is not None:
234
+ self.df = st.session_state["response"]
235
+
236
+ if st.session_state["action"] == "delete" and "response" in st.session_state:
237
+ if st.session_state["response"] is not None:
238
+ self.df = st.session_state["response"]
239
+
240
+ response = agstyler.draw_grid(
241
+ self.df,
242
+ formatter=formatter,
243
+ fit_columns=True,
244
+ pagination_size=10,
245
+ selection="single",
246
+ use_checkbox=False,
247
+ )
248
+
249
+ rows = response["selected_rows"]
250
+ st.session_state["selected_rows"] = rows
251
+
252
+ if st.session_state["action"] == "create" and self.action_event:
253
+ st.session_state["response"] = response["data"]
254
+ elif st.session_state["action"] == "delete" and self.action_event:
255
+ st.session_state["response"] = response["data"]
256
+ elif st.session_state["action"] == "save" and self.action_event:
257
+ data = response["data"].values.tolist()
258
+ rows = []
259
+ for row in data:
260
+ rows.append({"id": row[0], "name": row[1], "description": row[2]})
261
+
262
+ groups_json["groups"] = rows
263
+ with open(model.groups_file, "w") as f:
264
+ json.dump(groups_json, f, indent=2)
265
+
266
+ def setup_subgroups(self, model):
267
+ self.action_event = False
268
+ if "action" not in st.session_state:
269
+ st.session_state["action"] = None
270
+
271
+ with open(model.subgroups_file, "r") as f:
272
+ subgroups_json = json.load(f)
273
+
274
+ subgroups = subgroups_json["subgroups"]
275
+
276
+ data = []
277
+ for subgroup in subgroups:
278
+ data.append(
279
+ {
280
+ "id": subgroup["id"],
281
+ "name": subgroup["name"],
282
+ "description": subgroup["description"],
283
+ }
284
+ )
285
+ self.df = pd.DataFrame(data)
286
+
287
+ formatter = {
288
+ "id": ("ID", {"hide": True}),
289
+ "name": ("Sub Group", {**PINLEFT, "editable": True}),
290
+ "description": ("Description", {**PINLEFT, "editable": True}),
291
+ }
292
+
293
+ def run_component(props):
294
+ value = component_toolbar_buttons(key="toolbar_buttons_groups", **props)
295
+ return value
296
+
297
+ def handle_event(value):
298
+ if value is not None:
299
+ if "action_timestamp" not in st.session_state:
300
+ self.action_event = True
301
+ st.session_state["action_timestamp"] = value["timestamp"]
302
+ else:
303
+ if st.session_state["action_timestamp"] != value["timestamp"]:
304
+ self.action_event = True
305
+ st.session_state["action_timestamp"] = value["timestamp"]
306
+ else:
307
+ self.action_event = False
308
+
309
+ if value is not None and value["action"] == "create" and self.action_event:
310
+ if st.session_state["action"] != "delete":
311
+ max_id = self.df["id"].max()
312
+ self.df.loc[-1] = [max_id + 1, "", ""] # adding a row
313
+ self.df.index = self.df.index + 1 # shifting index
314
+ self.df.sort_index(inplace=True)
315
+ st.session_state["action"] = "create"
316
+ elif (
317
+ value is not None and value["action"] == "delete" and self.action_event
318
+ ):
319
+ if (
320
+ st.session_state["action"] != "delete"
321
+ and st.session_state["action"] != "create"
322
+ ):
323
+ rows = st.session_state["selected_rows"]
324
+ if len(rows) > 0:
325
+ idx = rows[0]["_selectedRowNodeInfo"]["nodeRowIndex"]
326
+ self.df.drop(self.df.index[idx], inplace=True)
327
+ self.df.reset_index(drop=True, inplace=True)
328
+ st.session_state["action"] = "delete"
329
+ elif value is not None and value["action"] == "save" and self.action_event:
330
+ st.session_state["action"] = "save"
331
+
332
+ props = {
333
+ "buttons": {
334
+ "create": False,
335
+ "delete": False,
336
+ "save": False,
337
+ }
338
+ }
339
+
340
+ handle_event(run_component(props))
341
+
342
+ if st.session_state["action"] == "save" and "response" in st.session_state:
343
+ if st.session_state["response"] is not None:
344
+ self.df = st.session_state["response"]
345
+ st.session_state["response"] = None
346
+
347
+ if st.session_state["action"] == "create" and "response" in st.session_state:
348
+ if st.session_state["response"] is not None:
349
+ self.df = st.session_state["response"]
350
+
351
+ if st.session_state["action"] == "delete" and "response" in st.session_state:
352
+ if st.session_state["response"] is not None:
353
+ self.df = st.session_state["response"]
354
+
355
+ response = agstyler.draw_grid(
356
+ self.df,
357
+ formatter=formatter,
358
+ fit_columns=True,
359
+ pagination_size=10,
360
+ selection="single",
361
+ use_checkbox=False,
362
+ )
363
+
364
+ rows = response["selected_rows"]
365
+ st.session_state["selected_rows"] = rows
366
+
367
+ if st.session_state["action"] == "create" and self.action_event:
368
+ st.session_state["response"] = response["data"]
369
+ elif st.session_state["action"] == "delete" and self.action_event:
370
+ st.session_state["response"] = response["data"]
371
+ elif st.session_state["action"] == "save" and self.action_event:
372
+ data = response["data"].values.tolist()
373
+ rows = []
374
+ for row in data:
375
+ rows.append({"id": row[0], "name": row[1], "description": row[2]})
376
+
377
+ subgroups_json["subgroups"] = rows
378
+ with open(model.subgroups_file, "w") as f:
379
+ json.dump(subgroups_json, f, indent=2)