From 753e05dcbf0969ed88b88ee24409935a91f34799 Mon Sep 17 00:00:00 2001
From: Nicolas Ferre <nicolas.ferre@arhs-spikeseed.com>
Date: Thu, 19 Oct 2023 10:51:48 +0200
Subject: [PATCH] AITED-257: convert lawfulness to api
---
dev.tfvars | 7 +-
main.tf | 5 +
modules/classifiers/dashboard.tf | 12 +-
modules/classifiers/outputs.tf | 3 +
modules/classifiers/variables.tf | 4 +
modules/lawfulness/outputs.tf | 4 +
modules/lawfulness/processing_api.tf | 179 ++++++++++++++++++
modules/lawfulness/processing_common.tf | 96 ++++++++++
.../lawfulness/{ecs.tf => processing_task.tf} | 132 ++-----------
modules/lawfulness/variables.tf | 18 ++
variables.tf | 4 +
11 files changed, 344 insertions(+), 120 deletions(-)
create mode 100644 modules/classifiers/outputs.tf
create mode 100644 modules/lawfulness/processing_api.tf
create mode 100644 modules/lawfulness/processing_common.tf
rename modules/lawfulness/{ecs.tf => processing_task.tf} (65%)
diff --git a/dev.tfvars b/dev.tfvars
index 3332853..d8c0bea 100644
--- a/dev.tfvars
+++ b/dev.tfvars
@@ -45,7 +45,7 @@ sagemaker_classifier_budgetary_value_classifier_model_url = "s3://d-ew1-ted
ecr_repository_application_name = "ted-applications"
ecr_repository_sagemaker_classifiers_name = "sagemaker-classifiers"
-applications_dashboard_repository_url = "dkr.ecr.eu-west-1.amazonaws.com/ted-applications:dashboard-v0.0.5"
+applications_dashboard_repository_url = "dkr.ecr.eu-west-1.amazonaws.com/ted-applications:dashboard-v0.0.7"
dashboard_port = 7860
# RDS DB
@@ -88,7 +88,8 @@ iam_policy_prefix = "D_EW1_TED_AI"
# Lawfulness
lawfulness_resource_prefix = "d-ew1-ted-ai-lawfulness"
-lawfulness_task_image = "528719223857.dkr.ecr.eu-west-1.amazonaws.com/ted-applications:lawfulness-v0.1.0"
+lawfulness_task_image = "528719223857.dkr.ecr.eu-west-1.amazonaws.com/ted-applications:lawfulness-v0.2.0"
+lawfulness_api_port = 8080
lawfulness_thread_count = 2
# Ingestion
@@ -97,4 +98,4 @@ notice_ingestion_resources_url = "dkr.ecr.eu-west-1.amazonaws.com/ted-applicatio
# Notice data extraction
notice_data_extraction_prefix = "d-ew1-ted-ai-notice-data-extraction"
-notice_data_extraction_resources_url = "dkr.ecr.eu-west-1.amazonaws.com/ted-applications:notice_data_extraction-v0.0.1"
\ No newline at end of file
+notice_data_extraction_resources_url = "dkr.ecr.eu-west-1.amazonaws.com/ted-applications:notice_data_extraction-v0.0.1"
diff --git a/main.tf b/main.tf
index 516b9cf..6ad72da 100644
--- a/main.tf
+++ b/main.tf
@@ -72,6 +72,7 @@ module "classifiers" {
sagemaker_classifier_budgetary_value_classifier_name = var.sagemaker_classifier_budgetary_value_classifier_name
ssm_classifier_endpoint_budgetary_value_classifier_name = var.ssm_classifier_endpoint_budgetary_value_classifier_name
project_account_id = var.project_account_id
+ lawfulness_host = module.lawfulness.processing_api_host
}
module "ingestion" {
@@ -138,18 +139,22 @@ module "lawfulness" {
tags = var.tags
vpc_id = var.vpc_id
private_subnet_id_list = var.private_subnet_id_list
+ private_subnet_id_az1_list = var.private_subnet_id_az1_list
+ private_subnet_id_az2_list = var.private_subnet_id_az2_list
iam_policy_prefix = var.iam_policy_prefix
iam_role_prefix = var.iam_role_prefix
ssm_prefix = var.ssm_prefix
resource_prefix = var.lawfulness_resource_prefix
task_image = var.lawfulness_task_image
input_bucket_arn = module.storage.tedai_storage_s3_buckets_map.input_bucket.arn
+ dashboard_security_group_id = module.classifiers.dashboard_security_group_id
db_name = var.db_name
db_username = var.db_username
db_host = module.db.host
db_port = module.db.port
db_password_ssm_parameter_arn = module.db.password_ssm_parameter_arn
db_password_ssm_parameter_name = module.db.password_ssm_parameter_name
+ api_port = var.lawfulness_api_port
thread_count = var.lawfulness_thread_count
}
diff --git a/modules/classifiers/dashboard.tf b/modules/classifiers/dashboard.tf
index 7bef1fa..3f144ca 100644
--- a/modules/classifiers/dashboard.tf
+++ b/modules/classifiers/dashboard.tf
@@ -53,8 +53,10 @@ resource "aws_iam_policy" "dashboard" {
Resource = "*"
},
{
- Action = ["sagemaker:InvokeEndpoint", "sagemaker:CreateEndpoint", "sagemaker:DeleteEndpoint",
- "sagemaker:DescribeEndpoint"]
+ Action = [
+ "sagemaker:InvokeEndpoint", "sagemaker:CreateEndpoint", "sagemaker:DeleteEndpoint",
+ "sagemaker:DescribeEndpoint"
+ ]
Effect = "Allow"
Resource = "*"
},
@@ -115,6 +117,12 @@ resource "aws_ecs_task_definition" "dashboard" {
command = ["CMD-SHELL", format("curl -f http://localhost:%s/ || exit 1", var.dashboard_port)]
startPeriod = 10
}
+ environment = [
+ {
+ name = "LAWFULNESS_API_URL"
+ value = "http://${var.lawfulness_host}"
+ }
+ ]
}
])
}
diff --git a/modules/classifiers/outputs.tf b/modules/classifiers/outputs.tf
new file mode 100644
index 0000000..5923332
--- /dev/null
+++ b/modules/classifiers/outputs.tf
@@ -0,0 +1,3 @@
+output "dashboard_security_group_id" {
+ value = aws_security_group.ecs_tasks.id
+}
\ No newline at end of file
diff --git a/modules/classifiers/variables.tf b/modules/classifiers/variables.tf
index 8e811e7..376973a 100644
--- a/modules/classifiers/variables.tf
+++ b/modules/classifiers/variables.tf
@@ -46,6 +46,10 @@ variable "dashboard_port" {
type = number
}
+variable "lawfulness_host" {
+ type = string
+}
+
# Network
variable "vpc_id" {
type = string
diff --git a/modules/lawfulness/outputs.tf b/modules/lawfulness/outputs.tf
index 8a0a0e2..a193796 100644
--- a/modules/lawfulness/outputs.tf
+++ b/modules/lawfulness/outputs.tf
@@ -1,3 +1,7 @@
output "new_notices_queue_arn" {
value = aws_sqs_queue.new_notices.arn
}
+
+output "processing_api_host" {
+ value = aws_alb.processing_api.dns_name
+}
diff --git a/modules/lawfulness/processing_api.tf b/modules/lawfulness/processing_api.tf
new file mode 100644
index 0000000..4ffe47e
--- /dev/null
+++ b/modules/lawfulness/processing_api.tf
@@ -0,0 +1,179 @@
+resource "aws_iam_policy" "processing_api" {
+ name = "${var.iam_policy_prefix}_LAWFULNESS_PROCESSING_API_POLICY"
+ policy = jsonencode({
+ Version = "2012-10-17"
+ Statement = [
+ {
+ Action = ["ssm:GetParameter"]
+ Effect = "Allow"
+ Resource = var.db_password_ssm_parameter_arn
+ },
+ {
+ Action = ["logs:CreateLogStream", "logs:PutLogEvents"]
+ Effect = "Allow"
+ Resource = "${aws_cloudwatch_log_group.cluster.arn}:*"
+ },
+ {
+ Action = ["ecr:BatchGetImage*", "ecr:BatchCheck*", "ecr:Get*", "ecr:List*", "ecr:Describe*"]
+ Effect = "Allow"
+ Resource = "*"
+ }
+ ]
+ })
+}
+
+resource "aws_iam_role" "processing_api" {
+ name = "${var.iam_role_prefix}_LAWFULNESS_PROCESSING_API_ROLE"
+ assume_role_policy = data.aws_iam_policy_document.ecs_assume_role_policy.json
+ managed_policy_arns = [aws_iam_policy.processing_api.arn]
+ permissions_boundary = "arn:aws:iam::528719223857:policy/Team_Admin_Boundary"
+}
+
+
+resource "aws_ecs_task_definition" "processing_api" {
+ family = "${var.resource_prefix}-processing-api"
+ requires_compatibilities = ["FARGATE"]
+ network_mode = "awsvpc"
+ cpu = 1024
+ memory = 2048
+ execution_role_arn = aws_iam_role.processing_api.arn
+ task_role_arn = aws_iam_role.processing_api.arn
+ container_definitions = jsonencode([
+ {
+ name = "lawfulness"
+ image = var.task_image
+ cpu = 1024
+ memory = 2048
+ execution_role_arn = aws_iam_role.processing_api.arn
+ task_role_arn = aws_iam_role.processing_api.arn
+ essential = true
+ logConfiguration = {
+ logDriver = "awslogs"
+ options = {
+ awslogs-group = aws_cloudwatch_log_group.cluster.name
+ awslogs-region = var.region
+ awslogs-stream-prefix = "ecs"
+ }
+ }
+ portMappings = [
+ {
+ containerPort = var.api_port,
+ hostPort = var.api_port
+ }
+ ]
+ healthCheck = {
+ command = ["CMD-SHELL", "curl -f http://localhost:${var.api_port}/health || exit 1"]
+ startPeriod = 10
+ }
+ environment = concat(local.task_environment, [{ name = "MODE", value = "api" }])
+ }
+ ])
+}
+
+resource "aws_security_group" "processing_api" {
+ name = "${var.resource_prefix}-processing-api"
+ description = "Allow internet and service port access in lawfulness processing API"
+ vpc_id = var.vpc_id
+
+ ingress {
+ protocol = "tcp"
+ from_port = var.api_port
+ to_port = var.api_port
+ security_groups = [aws_security_group.processing_api_load_balancer.id]
+ }
+
+ egress {
+ protocol = "-1"
+ from_port = 0
+ to_port = 0
+ cidr_blocks = ["0.0.0.0/0"]
+ }
+}
+
+resource "aws_ecs_service" "processing_api" {
+ name = "${var.resource_prefix}-processing-api"
+ cluster = aws_ecs_cluster.cluster.id
+ task_definition = aws_ecs_task_definition.processing_api.arn
+ desired_count = 1
+ launch_type = "FARGATE"
+
+ network_configuration {
+ security_groups = [aws_security_group.processing_api.id]
+ subnets = var.private_subnet_id_list
+ assign_public_ip = false
+ }
+
+ load_balancer {
+ target_group_arn = aws_alb_target_group.processing_api.id
+ container_name = "lawfulness"
+ container_port = var.api_port
+ }
+ depends_on = [aws_iam_role.processing_api, aws_alb.processing_api]
+}
+
+resource "aws_security_group" "processing_api_load_balancer" {
+ name = "${var.resource_prefix}-processing-api-load-lalancer"
+ description = "Controls access to the ALB of lawfulness API"
+ vpc_id = var.vpc_id
+
+ ingress {
+ protocol = "tcp"
+ from_port = 80
+ to_port = 80
+ security_groups = [var.dashboard_security_group_id]
+ }
+
+ egress {
+ protocol = "-1"
+ from_port = 0
+ to_port = 0
+ cidr_blocks = ["0.0.0.0/0"]
+ }
+}
+
+resource "random_shuffle" "az1" {
+ input = var.private_subnet_id_az1_list
+ result_count = 1
+}
+
+resource "random_shuffle" "az2" {
+ input = var.private_subnet_id_az2_list
+ result_count = 1
+}
+
+resource "aws_alb" "processing_api" {
+ name = "lawfulness-processing-api"
+ subnets = concat(random_shuffle.az1.result, random_shuffle.az2.result)
+ security_groups = [aws_security_group.processing_api_load_balancer.id]
+ internal = true
+}
+
+resource "aws_alb_target_group" "processing_api" {
+ name = "lawfulness-processing-api"
+ port = var.api_port
+ protocol = "HTTP"
+ vpc_id = var.vpc_id
+ target_type = "ip"
+
+ health_check {
+ healthy_threshold = "3"
+ interval = "30"
+ protocol = "HTTP"
+ matcher = "200"
+ timeout = "3"
+ path = "/health"
+ unhealthy_threshold = "3"
+ }
+}
+
+# Redirect all traffic from the ALB to the target group
+resource "aws_alb_listener" "processing_api" {
+ load_balancer_arn = aws_alb.processing_api.id
+ port = 80
+ protocol = "HTTP"
+
+ default_action {
+ target_group_arn = aws_alb_target_group.processing_api.id
+ type = "forward"
+ }
+}
diff --git a/modules/lawfulness/processing_common.tf b/modules/lawfulness/processing_common.tf
new file mode 100644
index 0000000..bb63f85
--- /dev/null
+++ b/modules/lawfulness/processing_common.tf
@@ -0,0 +1,96 @@
+resource "aws_ecs_cluster" "cluster" {
+ name = var.resource_prefix
+
+ setting {
+ name = "containerInsights"
+ value = "enabled"
+ }
+
+ configuration {
+ execute_command_configuration {
+ logging = "OVERRIDE"
+ log_configuration {
+ cloud_watch_log_group_name = aws_cloudwatch_log_group.cluster.name
+ }
+ }
+ }
+}
+
+resource "aws_cloudwatch_log_group" "cluster" {
+ name = "/tedai/lawfulness"
+}
+
+resource "aws_ecs_cluster_capacity_providers" "cluster" {
+ cluster_name = aws_ecs_cluster.cluster.name
+ capacity_providers = ["FARGATE"]
+
+ default_capacity_provider_strategy {
+ base = 1
+ weight = 100
+ capacity_provider = "FARGATE"
+ }
+}
+
+data "aws_iam_policy_document" "ecs_assume_role_policy" {
+ statement {
+ actions = ["sts:AssumeRole"]
+
+ principals {
+ type = "Service"
+ identifiers = ["ecs.amazonaws.com", "ecs-tasks.amazonaws.com"]
+ }
+ }
+}
+
+locals {
+ task_environment = [
+ {
+ name = "LOG_LEVEL"
+ value = "DEBUG"
+ },
+ {
+ name = "THREAD_COUNT"
+ value = tostring(var.thread_count)
+ },
+ {
+ name = "FLAGGED_NOTICES_DYNAMODB_TABLE_NAME"
+ value = aws_dynamodb_table.flagged_notices.name
+ },
+ {
+ name = "MAX_WHITELISTED_OFFICIAL_NAME_SIMILARITY"
+ value = "0.8"
+ },
+ {
+ name = "DB_HOST"
+ value = var.db_host
+ },
+ {
+ name = "DB_PORT"
+ value = var.db_port
+ },
+ {
+ name = "DB_NAME"
+ value = var.db_name
+ },
+ {
+ name = "DB_USERNAME"
+ value = var.db_username
+ },
+ {
+ name = "DB_PASSWORD_SSM_PARAMETER"
+ value = var.db_password_ssm_parameter_name
+ },
+ {
+ name = "DB_WHITELISTED_BODIES_TABLE"
+ value = "whitelisted_contracting_bodies"
+ },
+ {
+ name = "NEW_NOTICES_QUEUE_URL"
+ value = aws_sqs_queue.new_notices.url
+ },
+ {
+ name = "NEW_NOTICES_BATCH_SIZE"
+ value = "10"
+ },
+ ]
+}
diff --git a/modules/lawfulness/ecs.tf b/modules/lawfulness/processing_task.tf
similarity index 65%
rename from modules/lawfulness/ecs.tf
rename to modules/lawfulness/processing_task.tf
index 5e688c5..49ab747 100644
--- a/modules/lawfulness/ecs.tf
+++ b/modules/lawfulness/processing_task.tf
@@ -1,47 +1,3 @@
-resource "aws_ecs_cluster" "cluster" {
- name = var.resource_prefix
-
- setting {
- name = "containerInsights"
- value = "enabled"
- }
-
- configuration {
- execute_command_configuration {
- logging = "OVERRIDE"
- log_configuration {
- cloud_watch_log_group_name = aws_cloudwatch_log_group.cluster.name
- }
- }
- }
-}
-
-resource "aws_cloudwatch_log_group" "cluster" {
- name = "/tedai/lawfulness"
-}
-
-resource "aws_ecs_cluster_capacity_providers" "cluster" {
- cluster_name = aws_ecs_cluster.cluster.name
- capacity_providers = ["FARGATE"]
-
- default_capacity_provider_strategy {
- base = 1
- weight = 100
- capacity_provider = "FARGATE"
- }
-}
-
-data "aws_iam_policy_document" "ecs_assume_role_policy" {
- statement {
- actions = ["sts:AssumeRole"]
-
- principals {
- type = "Service"
- identifiers = ["ecs.amazonaws.com", "ecs-tasks.amazonaws.com"]
- }
- }
-}
-
resource "aws_iam_policy" "processing_task" {
name = "${var.iam_policy_prefix}_LAWFULNESS_PROCESSING_TASK_POLICY"
policy = jsonencode({
@@ -57,11 +13,6 @@ resource "aws_iam_policy" "processing_task" {
Effect = "Allow"
Resource = "${var.input_bucket_arn}/*"
},
- {
- Action = ["sqs:ReceiveMessage", "sqs:DeleteMessage"]
- Effect = "Allow"
- Resource = aws_sqs_queue.new_notices.arn
- },
{
Action = ["dynamodb:PutItem"]
Effect = "Allow"
@@ -93,6 +44,18 @@ resource "aws_iam_role" "processing_task" {
permissions_boundary = "arn:aws:iam::528719223857:policy/Team_Admin_Boundary"
}
+resource "aws_security_group" "processing_task" {
+ name = "${var.resource_prefix}-processing-task"
+ description = "Allow internet access in lawfulness processing task"
+ vpc_id = var.vpc_id
+
+ egress {
+ protocol = "-1"
+ from_port = 0
+ to_port = 0
+ cidr_blocks = ["0.0.0.0/0"]
+ }
+}
resource "aws_ecs_task_definition" "processing_task" {
family = "${var.resource_prefix}-processing-task"
@@ -119,73 +82,11 @@ resource "aws_ecs_task_definition" "processing_task" {
awslogs-stream-prefix = "ecs"
}
}
- environment = [
- {
- name = "LOG_LEVEL"
- value = "DEBUG"
- },
- {
- name = "THREAD_COUNT"
- value = tostring(var.thread_count)
- },
- {
- name = "NEW_NOTICES_QUEUE_URL"
- value = aws_sqs_queue.new_notices.url
- },
- {
- name = "NEW_NOTICES_BATCH_SIZE"
- value = "10"
- },
- {
- name = "FLAGGED_NOTICES_DYNAMODB_TABLE_NAME"
- value = aws_dynamodb_table.flagged_notices.name
- },
- {
- name = "DB_HOST"
- value = var.db_host
- },
- {
- name = "DB_PORT"
- value = var.db_port
- },
- {
- name = "DB_NAME"
- value = var.db_name
- },
- {
- name = "DB_USERNAME"
- value = var.db_username
- },
- {
- name = "DB_PASSWORD_SSM_PARAMETER"
- value = var.db_password_ssm_parameter_name
- },
- {
- name = "WHITELISTED_BODIES_TABLE_NAME"
- value = "whitelisted_contracting_bodies"
- },
- {
- name = "MAX_WHITELISTED_OFFICIAL_NAME_SIMILARITY"
- value = "0.8"
- },
- ],
+ environment = concat(local.task_environment, [{ name = "MODE", value = "task" }])
}
])
}
-resource "aws_security_group" "processing" {
- name = "${var.resource_prefix}-processing-task"
- description = "Allow internet access in lawfulness processing task"
- vpc_id = var.vpc_id
-
- egress {
- protocol = "-1"
- from_port = 0
- to_port = 0
- cidr_blocks = ["0.0.0.0/0"]
- }
-}
-
resource "aws_scheduler_schedule" "processing_scheduler" {
name = "${var.resource_prefix}-processing-scheduler"
group_name = "default"
@@ -204,7 +105,7 @@ resource "aws_scheduler_schedule" "processing_scheduler" {
launch_type = "FARGATE"
network_configuration {
- security_groups = [aws_security_group.processing.id]
+ security_groups = [aws_security_group.processing_task.id]
subnets = var.private_subnet_id_list
assign_public_ip = false
}
@@ -217,6 +118,7 @@ resource "aws_scheduler_schedule" "processing_scheduler" {
}
}
+
data "aws_iam_policy_document" "scheduler_assume_role_policy" {
statement {
actions = ["sts:AssumeRole"]
@@ -236,12 +138,12 @@ resource "aws_iam_policy" "processing_scheduler" {
{
Action = ["ecs:RunTask"]
Effect = "Allow"
- Resource = aws_ecs_task_definition.processing_task.arn_without_revision
+ Resource = [aws_ecs_task_definition.processing_task.arn_without_revision]
},
{
Action = ["iam:PassRole"]
Effect = "Allow"
- Resource = aws_iam_role.processing_task.arn
+ Resource = [aws_iam_role.processing_task.arn]
}
]
})
diff --git a/modules/lawfulness/variables.tf b/modules/lawfulness/variables.tf
index e0cf895..50d4c6c 100644
--- a/modules/lawfulness/variables.tf
+++ b/modules/lawfulness/variables.tf
@@ -14,6 +14,14 @@ variable "private_subnet_id_list" {
type = list(string)
}
+variable "private_subnet_id_az1_list" {
+ type = list(string)
+}
+
+variable "private_subnet_id_az2_list" {
+ type = list(string)
+}
+
variable "iam_policy_prefix" {
type = string
}
@@ -38,6 +46,10 @@ variable "input_bucket_arn" {
type = string
}
+variable "dashboard_security_group_id" {
+ type = string
+}
+
variable "db_host" {
type = string
}
@@ -62,6 +74,12 @@ variable "db_password_ssm_parameter_arn" {
type = string
}
+
+variable "api_port" {
+ type = number
+}
+
variable "thread_count" {
type = number
}
+
diff --git a/variables.tf b/variables.tf
index 3cd2d82..ee0c4b9 100644
--- a/variables.tf
+++ b/variables.tf
@@ -248,6 +248,10 @@ variable "lawfulness_task_image" {
type = string
}
+variable "lawfulness_api_port" {
+ type = number
+}
+
variable "lawfulness_thread_count" {
type = number
}
--
GitLab