AITED-257: add lawfulness

Dockerfile

@@ -6,7 +6,7 @@ COPY requirements.txt requirements.txt
 RUN pip3 install -r requirements.txt
 
 WORKDIR /app
-COPY app/dashboard.py /app/dashboard.py
+COPY app /app
 
 EXPOSE 7860
 

README.md

@@ -32,8 +32,8 @@ Replace xxx by the AWS profile of the TED AI environment
 
 ## CI/CD
 TED AI AWS environment contains three Amazon ECR repositories:
-* sagemaker-classifiers-ci: for all docker images pushed during CI setup for testing. These images are automatically deleted after 1 day
+* ci-temporary-images: for all docker images pushed during CI setup for testing. These images are automatically deleted after 1 day
 * sagemaker-classifiers: for all docker images of models meant to be used in production after the testing was completed
 * ted-applications: for docker images of applications
 
-This pipeline deploys in sagemaker-classifiers-ci when in a feature branch or main. When a tag is created, the docker image is pushed to ted-applications.
+This pipeline deploys in ci-temporary-images when in a feature branch or main. When a tag is created, the docker image is pushed to ted-applications.

app/dashboard.py

@@ -6,6 +6,8 @@ import gradio as gr
 import pandas as pd
 import urllib3
 
+import lawfulness
+
 sagemaker_runtime_client = boto3.client('sagemaker-runtime', region_name="eu-west-1")
 sagemaker_client = boto3.client('sagemaker', region_name="eu-west-1")
 ssm_client = boto3.client('ssm', region_name="eu-west-1")
@@ -229,4 +231,6 @@ if __name__ == '__main__':
             cbve_predict_button.click(model_budgetary_value_classifier_file, inputs=cbve_file,
                                       outputs=cbve_file_output_textbox)
 
+        lawfulness.display_tab()
+
 demo.launch(server_name="0.0.0.0", server_port=7860)

app/lawfulness.py

+import os
+from pathlib import Path
+from typing import List
+
+import gradio as gr
+import requests
+from requests import Response
+
+API_URL = os.environ.get("LAWFULNESS_API_URL", "http://localhost:8080")
+
+STATUS_COLUMNS = [
+    'Filename',
+    'Supported',
+    'Notice number',
+    'Must be reviewed',
+]
+FLAGS_COLUMNS = [
+    'Notice number',
+    '"test" word',
+    '"sale" word',
+    'Incoherent text',
+    'Forbidden country',
+    'Non EU country + not funded by EU',
+    'Private contracting body + not funded by EU',
+    'Not whitelisted + international contracting body',
+]
+
+
+def display_tab():
+    with gr.Tab("Lawfulness"):
+        gr.Markdown("# <u>Lawfulness</u>")
+        gr.Markdown("Only eForms notices in XML format are supported.")
+        files = gr.File(file_count="multiple")
+        run_button = gr.Button("Run lawfulness processing")
+        status_table = gr.Dataframe(row_count=(3, "dynamic"),
+                                    col_count=(len(STATUS_COLUMNS), "fixed"),
+                                    headers=STATUS_COLUMNS,
+                                    label="Status",
+                                    height=200)
+        flags_table = gr.Dataframe(row_count=(3, "dynamic"),
+                                   col_count=(len(FLAGS_COLUMNS), "fixed"),
+                                   headers=FLAGS_COLUMNS,
+                                   label="Notices to review",
+                                   height=200)
+        show_raw_checkbox = gr.Checkbox(label="Show raw response")
+        details_block = gr.JSON(label="Raw response", visible=False)
+        run_button.click(_run, inputs=files, outputs=[status_table, flags_table, details_block])
+        show_raw_checkbox.change(_show_raw, inputs=[show_raw_checkbox], outputs=[details_block])
+
+
+def _run(files: list) -> List[list]:
+    response = requests.post(f"{API_URL}/notices/lawfulness", files=[
+        ("file", Path(file.name).read_bytes())
+        for file in files
+    ])
+    return [_format_status(files, response), _format_flags(files, response), response.text]
+
+
+def _format_status(files: list, response: Response):
+    results = []
+    for file, result in zip(files, response.json()):
+        results.append([
+            Path(file.name).name,
+            bool(result),
+            result['notice_id'] if result else "-",
+            any(result['flags'].values()) if result else "-",
+        ])
+    return results
+
+
+def _format_flags(files: list, response: Response):
+    results = []
+    for file, result in zip(files, response.json()):
+        if not result or not any(result['flags'].values()):
+            continue
+        results.append([
+            result['notice_id'],
+            result['flags']['has_test_word'],
+            result['flags']['has_sale_word'],
+            result['flags']['has_incoherent_text'],
+            result['flags']['is_forbidden_country'],
+            result['flags']['has_non_eu_country_and_not_funded_by_eu'],
+            result['flags']['is_contracting_body_private_and_not_funded_by_eu'],
+            result['flags']['has_not_whitelisted_international_contracting_body'],
+        ])
+    return results
+
+
+def _show_raw(is_shown: bool) -> gr.JSON:
+    return gr.JSON(visible=is_shown)

requirements.txt

-aiofiles==23.1.0
-aiohttp==3.8.4
-aiosignal==1.3.1
-altair==4.2.2
-anyio==3.6.2
-async-timeout==4.0.2
-attrs==23.1.0
-boto3==1.26.114
-botocore==1.29.114
-certifi==2022.12.7
-charset-normalizer==3.1.0
-click==8.1.3
-contourpy==1.0.7
-cycler==0.11.0
-entrypoints==0.4
-fastapi==0.95.1
-ffmpy==0.3.0
-filelock==3.11.0
-fonttools==4.39.3
-frozenlist==1.3.3
-fsspec==2023.4.0
-gradio==3.27.0
-gradio_client==0.1.3
-h11==0.14.0
-httpcore==0.17.0
-httpx==0.24.0
-huggingface-hub==0.13.4
-idna==3.4
-Jinja2==3.1.2
-jmespath==1.0.1
-jsonschema==4.17.3
-kiwisolver==1.4.4
-linkify-it-py==2.0.0
-markdown-it-py==2.2.0
-MarkupSafe==2.1.2
-matplotlib==3.7.1
-mdit-py-plugins==0.3.3
-mdurl==0.1.2
-multidict==6.0.4
-numpy==1.24.2
-orjson==3.8.10
-packaging==23.1
-pandas==2.0.0
-Pillow==9.5.0
-pydantic==1.10.7
-pydub==0.25.1
-pyparsing==3.0.9
-pyrsistent==0.19.3
-python-dateutil==2.8.2
-python-multipart==0.0.6
-pytz==2023.3
-PyYAML==6.0
-requests==2.28.2
-s3transfer==0.6.0
-semantic-version==2.10.0
-six==1.16.0
-sniffio==1.3.0
-starlette==0.26.1
-toolz==0.12.0
-tqdm==4.65.0
-typing_extensions==4.5.0
-tzdata==2023.3
-uc-micro-py==1.0.1
-urllib3==1.26.15
-uvicorn==0.21.1
-websockets==11.0.1
-yarl==1.8.2
+boto3==1.28.65
+gradio==3.49.0
+pandas==2.0.3
+requests==2.31.0
+urllib3==1.26.18