AITED-257: convert lawfulness to api

dev.tfvars

@@ -45,7 +45,7 @@ sagemaker_classifier_budgetary_value_classifier_model_url      = "s3://d-ew1-ted
 ecr_repository_application_name           = "ted-applications"
 ecr_repository_sagemaker_classifiers_name = "sagemaker-classifiers"
 
-applications_dashboard_repository_url = "dkr.ecr.eu-west-1.amazonaws.com/ted-applications:dashboard-v0.0.5"
+applications_dashboard_repository_url = "dkr.ecr.eu-west-1.amazonaws.com/ted-applications:dashboard-v0.0.7"
 dashboard_port                        = 7860
 
 # RDS DB
@@ -88,7 +88,8 @@ iam_policy_prefix = "D_EW1_TED_AI"
 
 # Lawfulness
 lawfulness_resource_prefix = "d-ew1-ted-ai-lawfulness"
-lawfulness_task_image      = "528719223857.dkr.ecr.eu-west-1.amazonaws.com/ted-applications:lawfulness-v0.1.0"
+lawfulness_task_image      = "528719223857.dkr.ecr.eu-west-1.amazonaws.com/ted-applications:lawfulness-v0.2.0"
+lawfulness_api_port        = 8080
 lawfulness_thread_count    = 2
 
 # Ingestion
@@ -97,4 +98,4 @@ notice_ingestion_resources_url = "dkr.ecr.eu-west-1.amazonaws.com/ted-applicatio
 
 # Notice data extraction
 notice_data_extraction_prefix        = "d-ew1-ted-ai-notice-data-extraction"
-notice_data_extraction_resources_url = "dkr.ecr.eu-west-1.amazonaws.com/ted-applications:notice_data_extraction-v0.0.1"
\ No newline at end of file
+notice_data_extraction_resources_url = "dkr.ecr.eu-west-1.amazonaws.com/ted-applications:notice_data_extraction-v0.0.1"

main.tf

@@ -72,6 +72,7 @@ module "classifiers" {
   sagemaker_classifier_budgetary_value_classifier_name           = var.sagemaker_classifier_budgetary_value_classifier_name
   ssm_classifier_endpoint_budgetary_value_classifier_name        = var.ssm_classifier_endpoint_budgetary_value_classifier_name
   project_account_id                                             = var.project_account_id
+  lawfulness_host                                                = module.lawfulness.processing_api_host
 }
 
 module "ingestion" {
@@ -138,18 +139,22 @@ module "lawfulness" {
   tags                           = var.tags
   vpc_id                         = var.vpc_id
   private_subnet_id_list         = var.private_subnet_id_list
+  private_subnet_id_az1_list     = var.private_subnet_id_az1_list
+  private_subnet_id_az2_list     = var.private_subnet_id_az2_list
   iam_policy_prefix              = var.iam_policy_prefix
   iam_role_prefix                = var.iam_role_prefix
   ssm_prefix                     = var.ssm_prefix
   resource_prefix                = var.lawfulness_resource_prefix
   task_image                     = var.lawfulness_task_image
   input_bucket_arn               = module.storage.tedai_storage_s3_buckets_map.input_bucket.arn
+  dashboard_security_group_id    = module.classifiers.dashboard_security_group_id
   db_name                        = var.db_name
   db_username                    = var.db_username
   db_host                        = module.db.host
   db_port                        = module.db.port
   db_password_ssm_parameter_arn  = module.db.password_ssm_parameter_arn
   db_password_ssm_parameter_name = module.db.password_ssm_parameter_name
+  api_port                       = var.lawfulness_api_port
   thread_count                   = var.lawfulness_thread_count
 }
 

modules/classifiers/dashboard.tf

@@ -53,8 +53,10 @@ resource "aws_iam_policy" "dashboard" {
         Resource = "*"
       },
       {
-        Action = ["sagemaker:InvokeEndpoint", "sagemaker:CreateEndpoint", "sagemaker:DeleteEndpoint",
-        "sagemaker:DescribeEndpoint"]
+        Action = [
+          "sagemaker:InvokeEndpoint", "sagemaker:CreateEndpoint", "sagemaker:DeleteEndpoint",
+          "sagemaker:DescribeEndpoint"
+        ]
         Effect   = "Allow"
         Resource = "*"
       },
@@ -115,6 +117,12 @@ resource "aws_ecs_task_definition" "dashboard" {
         command     = ["CMD-SHELL", format("curl -f http://localhost:%s/ || exit 1", var.dashboard_port)]
         startPeriod = 10
       }
+      environment = [
+        {
+          name  = "LAWFULNESS_API_URL"
+          value = "http://${var.lawfulness_host}"
+        }
+      ]
     }
   ])
 }

modules/classifiers/outputs.tf

+output "dashboard_security_group_id" {
+  value = aws_security_group.ecs_tasks.id
+}
\ No newline at end of file

modules/classifiers/variables.tf

@@ -46,6 +46,10 @@ variable "dashboard_port" {
   type = number
 }
 
+variable "lawfulness_host" {
+  type = string
+}
+
 # Network
 variable "vpc_id" {
   type = string

modules/lawfulness/outputs.tf

 output "new_notices_queue_arn" {
   value = aws_sqs_queue.new_notices.arn
 }
+
+output "processing_api_host" {
+  value = aws_alb.processing_api.dns_name
+}

modules/lawfulness/processing_api.tf

+resource "aws_iam_policy" "processing_api" {
+  name = "${var.iam_policy_prefix}_LAWFULNESS_PROCESSING_API_POLICY"
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action   = ["ssm:GetParameter"]
+        Effect   = "Allow"
+        Resource = var.db_password_ssm_parameter_arn
+      },
+      {
+        Action   = ["logs:CreateLogStream", "logs:PutLogEvents"]
+        Effect   = "Allow"
+        Resource = "${aws_cloudwatch_log_group.cluster.arn}:*"
+      },
+      {
+        Action   = ["ecr:BatchGetImage*", "ecr:BatchCheck*", "ecr:Get*", "ecr:List*", "ecr:Describe*"]
+        Effect   = "Allow"
+        Resource = "*"
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role" "processing_api" {
+  name                 = "${var.iam_role_prefix}_LAWFULNESS_PROCESSING_API_ROLE"
+  assume_role_policy   = data.aws_iam_policy_document.ecs_assume_role_policy.json
+  managed_policy_arns  = [aws_iam_policy.processing_api.arn]
+  permissions_boundary = "arn:aws:iam::528719223857:policy/Team_Admin_Boundary"
+}
+
+
+resource "aws_ecs_task_definition" "processing_api" {
+  family                   = "${var.resource_prefix}-processing-api"
+  requires_compatibilities = ["FARGATE"]
+  network_mode             = "awsvpc"
+  cpu                      = 1024
+  memory                   = 2048
+  execution_role_arn       = aws_iam_role.processing_api.arn
+  task_role_arn            = aws_iam_role.processing_api.arn
+  container_definitions = jsonencode([
+    {
+      name               = "lawfulness"
+      image              = var.task_image
+      cpu                = 1024
+      memory             = 2048
+      execution_role_arn = aws_iam_role.processing_api.arn
+      task_role_arn      = aws_iam_role.processing_api.arn
+      essential          = true
+      logConfiguration = {
+        logDriver = "awslogs"
+        options = {
+          awslogs-group         = aws_cloudwatch_log_group.cluster.name
+          awslogs-region        = var.region
+          awslogs-stream-prefix = "ecs"
+        }
+      }
+      portMappings = [
+        {
+          containerPort = var.api_port,
+          hostPort      = var.api_port
+        }
+      ]
+      healthCheck = {
+        command     = ["CMD-SHELL", "curl -f http://localhost:${var.api_port}/health || exit 1"]
+        startPeriod = 10
+      }
+      environment = concat(local.task_environment, [{ name = "MODE", value = "api" }])
+    }
+  ])
+}
+
+resource "aws_security_group" "processing_api" {
+  name        = "${var.resource_prefix}-processing-api"
+  description = "Allow internet and service port access in lawfulness processing API"
+  vpc_id      = var.vpc_id
+
+  ingress {
+    protocol        = "tcp"
+    from_port       = var.api_port
+    to_port         = var.api_port
+    security_groups = [aws_security_group.processing_api_load_balancer.id]
+  }
+
+  egress {
+    protocol    = "-1"
+    from_port   = 0
+    to_port     = 0
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_ecs_service" "processing_api" {
+  name            = "${var.resource_prefix}-processing-api"
+  cluster         = aws_ecs_cluster.cluster.id
+  task_definition = aws_ecs_task_definition.processing_api.arn
+  desired_count   = 1
+  launch_type     = "FARGATE"
+
+  network_configuration {
+    security_groups  = [aws_security_group.processing_api.id]
+    subnets          = var.private_subnet_id_list
+    assign_public_ip = false
+  }
+
+  load_balancer {
+    target_group_arn = aws_alb_target_group.processing_api.id
+    container_name   = "lawfulness"
+    container_port   = var.api_port
+  }
+  depends_on = [aws_iam_role.processing_api, aws_alb.processing_api]
+}
+
+resource "aws_security_group" "processing_api_load_balancer" {
+  name        = "${var.resource_prefix}-processing-api-load-lalancer"
+  description = "Controls access to the ALB of lawfulness API"
+  vpc_id      = var.vpc_id
+
+  ingress {
+    protocol        = "tcp"
+    from_port       = 80
+    to_port         = 80
+    security_groups = [var.dashboard_security_group_id]
+  }
+
+  egress {
+    protocol    = "-1"
+    from_port   = 0
+    to_port     = 0
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "random_shuffle" "az1" {
+  input        = var.private_subnet_id_az1_list
+  result_count = 1
+}
+
+resource "random_shuffle" "az2" {
+  input        = var.private_subnet_id_az2_list
+  result_count = 1
+}
+
+resource "aws_alb" "processing_api" {
+  name            = "lawfulness-processing-api"
+  subnets         = concat(random_shuffle.az1.result, random_shuffle.az2.result)
+  security_groups = [aws_security_group.processing_api_load_balancer.id]
+  internal        = true
+}
+
+resource "aws_alb_target_group" "processing_api" {
+  name        = "lawfulness-processing-api"
+  port        = var.api_port
+  protocol    = "HTTP"
+  vpc_id      = var.vpc_id
+  target_type = "ip"
+
+  health_check {
+    healthy_threshold   = "3"
+    interval            = "30"
+    protocol            = "HTTP"
+    matcher             = "200"
+    timeout             = "3"
+    path                = "/health"
+    unhealthy_threshold = "3"
+  }
+}
+
+# Redirect all traffic from the ALB to the target group
+resource "aws_alb_listener" "processing_api" {
+  load_balancer_arn = aws_alb.processing_api.id
+  port              = 80
+  protocol          = "HTTP"
+
+  default_action {
+    target_group_arn = aws_alb_target_group.processing_api.id
+    type             = "forward"
+  }
+}

modules/lawfulness/processing_common.tf

+resource "aws_ecs_cluster" "cluster" {
+  name = var.resource_prefix
+
+  setting {
+    name  = "containerInsights"
+    value = "enabled"
+  }
+
+  configuration {
+    execute_command_configuration {
+      logging = "OVERRIDE"
+      log_configuration {
+        cloud_watch_log_group_name = aws_cloudwatch_log_group.cluster.name
+      }
+    }
+  }
+}
+
+resource "aws_cloudwatch_log_group" "cluster" {
+  name = "/tedai/lawfulness"
+}
+
+resource "aws_ecs_cluster_capacity_providers" "cluster" {
+  cluster_name       = aws_ecs_cluster.cluster.name
+  capacity_providers = ["FARGATE"]
+
+  default_capacity_provider_strategy {
+    base              = 1
+    weight            = 100
+    capacity_provider = "FARGATE"
+  }
+}
+
+data "aws_iam_policy_document" "ecs_assume_role_policy" {
+  statement {
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["ecs.amazonaws.com", "ecs-tasks.amazonaws.com"]
+    }
+  }
+}
+
+locals {
+  task_environment = [
+    {
+      name  = "LOG_LEVEL"
+      value = "DEBUG"
+    },
+    {
+      name  = "THREAD_COUNT"
+      value = tostring(var.thread_count)
+    },
+    {
+      name  = "FLAGGED_NOTICES_DYNAMODB_TABLE_NAME"
+      value = aws_dynamodb_table.flagged_notices.name
+    },
+    {
+      name  = "MAX_WHITELISTED_OFFICIAL_NAME_SIMILARITY"
+      value = "0.8"
+    },
+    {
+      name  = "DB_HOST"
+      value = var.db_host
+    },
+    {
+      name  = "DB_PORT"
+      value = var.db_port
+    },
+    {
+      name  = "DB_NAME"
+      value = var.db_name
+    },
+    {
+      name  = "DB_USERNAME"
+      value = var.db_username
+    },
+    {
+      name  = "DB_PASSWORD_SSM_PARAMETER"
+      value = var.db_password_ssm_parameter_name
+    },
+    {
+      name  = "DB_WHITELISTED_BODIES_TABLE"
+      value = "whitelisted_contracting_bodies"
+    },
+    {
+      name  = "NEW_NOTICES_QUEUE_URL"
+      value = aws_sqs_queue.new_notices.url
+    },
+    {
+      name  = "NEW_NOTICES_BATCH_SIZE"
+      value = "10"
+    },
+  ]
+}

modules/lawfulness/ecs.tf → modules/lawfulness/processing_task.tf

-resource "aws_ecs_cluster" "cluster" {
-  name = var.resource_prefix
-
-  setting {
-    name  = "containerInsights"
-    value = "enabled"
-  }
-
-  configuration {
-    execute_command_configuration {
-      logging = "OVERRIDE"
-      log_configuration {
-        cloud_watch_log_group_name = aws_cloudwatch_log_group.cluster.name
-      }
-    }
-  }
-}
-
-resource "aws_cloudwatch_log_group" "cluster" {
-  name = "/tedai/lawfulness"
-}
-
-resource "aws_ecs_cluster_capacity_providers" "cluster" {
-  cluster_name       = aws_ecs_cluster.cluster.name
-  capacity_providers = ["FARGATE"]
-
-  default_capacity_provider_strategy {
-    base              = 1
-    weight            = 100
-    capacity_provider = "FARGATE"
-  }
-}
-
-data "aws_iam_policy_document" "ecs_assume_role_policy" {
-  statement {
-    actions = ["sts:AssumeRole"]
-
-    principals {
-      type        = "Service"
-      identifiers = ["ecs.amazonaws.com", "ecs-tasks.amazonaws.com"]
-    }
-  }
-}
-
 resource "aws_iam_policy" "processing_task" {
   name = "${var.iam_policy_prefix}_LAWFULNESS_PROCESSING_TASK_POLICY"
   policy = jsonencode({
@@ -57,11 +13,6 @@ resource "aws_iam_policy" "processing_task" {
         Effect   = "Allow"
         Resource = "${var.input_bucket_arn}/*"
       },
-      {
-        Action   = ["sqs:ReceiveMessage", "sqs:DeleteMessage"]
-        Effect   = "Allow"
-        Resource = aws_sqs_queue.new_notices.arn
-      },
       {
         Action   = ["dynamodb:PutItem"]
         Effect   = "Allow"
@@ -93,6 +44,18 @@ resource "aws_iam_role" "processing_task" {
   permissions_boundary = "arn:aws:iam::528719223857:policy/Team_Admin_Boundary"
 }
 
+resource "aws_security_group" "processing_task" {
+  name        = "${var.resource_prefix}-processing-task"
+  description = "Allow internet access in lawfulness processing task"
+  vpc_id      = var.vpc_id
+
+  egress {
+    protocol    = "-1"
+    from_port   = 0
+    to_port     = 0
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
 
 resource "aws_ecs_task_definition" "processing_task" {
   family                   = "${var.resource_prefix}-processing-task"
@@ -119,73 +82,11 @@ resource "aws_ecs_task_definition" "processing_task" {
           awslogs-stream-prefix = "ecs"
         }
       }
-      environment = [
-        {
-          name  = "LOG_LEVEL"
-          value = "DEBUG"
-        },
-        {
-          name  = "THREAD_COUNT"
-          value = tostring(var.thread_count)
-        },
-        {
-          name  = "NEW_NOTICES_QUEUE_URL"
-          value = aws_sqs_queue.new_notices.url
-        },
-        {
-          name  = "NEW_NOTICES_BATCH_SIZE"
-          value = "10"
-        },
-        {
-          name  = "FLAGGED_NOTICES_DYNAMODB_TABLE_NAME"
-          value = aws_dynamodb_table.flagged_notices.name
-        },
-        {
-          name  = "DB_HOST"
-          value = var.db_host
-        },
-        {
-          name  = "DB_PORT"
-          value = var.db_port
-        },
-        {
-          name  = "DB_NAME"
-          value = var.db_name
-        },
-        {
-          name  = "DB_USERNAME"
-          value = var.db_username
-        },
-        {
-          name  = "DB_PASSWORD_SSM_PARAMETER"
-          value = var.db_password_ssm_parameter_name
-        },
-        {
-          name  = "WHITELISTED_BODIES_TABLE_NAME"
-          value = "whitelisted_contracting_bodies"
-        },
-        {
-          name  = "MAX_WHITELISTED_OFFICIAL_NAME_SIMILARITY"
-          value = "0.8"
-        },
-      ],
+      environment = concat(local.task_environment, [{ name = "MODE", value = "task" }])
     }
   ])
 }
 
-resource "aws_security_group" "processing" {
-  name        = "${var.resource_prefix}-processing-task"
-  description = "Allow internet access in lawfulness processing task"
-  vpc_id      = var.vpc_id
-
-  egress {
-    protocol    = "-1"
-    from_port   = 0
-    to_port     = 0
-    cidr_blocks = ["0.0.0.0/0"]
-  }
-}
-
 resource "aws_scheduler_schedule" "processing_scheduler" {
   name                = "${var.resource_prefix}-processing-scheduler"
   group_name          = "default"
@@ -204,7 +105,7 @@ resource "aws_scheduler_schedule" "processing_scheduler" {
       launch_type         = "FARGATE"
 
       network_configuration {
-        security_groups  = [aws_security_group.processing.id]
+        security_groups  = [aws_security_group.processing_task.id]
         subnets          = var.private_subnet_id_list
         assign_public_ip = false
       }
@@ -217,6 +118,7 @@ resource "aws_scheduler_schedule" "processing_scheduler" {
   }
 }
 
+
 data "aws_iam_policy_document" "scheduler_assume_role_policy" {
   statement {
     actions = ["sts:AssumeRole"]
@@ -236,12 +138,12 @@ resource "aws_iam_policy" "processing_scheduler" {
       {
         Action   = ["ecs:RunTask"]
         Effect   = "Allow"
-        Resource = aws_ecs_task_definition.processing_task.arn_without_revision
+        Resource = [aws_ecs_task_definition.processing_task.arn_without_revision]
       },
       {
         Action   = ["iam:PassRole"]
         Effect   = "Allow"
-        Resource = aws_iam_role.processing_task.arn
+        Resource = [aws_iam_role.processing_task.arn]
       }
     ]
   })

modules/lawfulness/variables.tf

@@ -14,6 +14,14 @@ variable "private_subnet_id_list" {
   type = list(string)
 }
 
+variable "private_subnet_id_az1_list" {
+  type = list(string)
+}
+
+variable "private_subnet_id_az2_list" {
+  type = list(string)
+}
+
 variable "iam_policy_prefix" {
   type = string
 }
@@ -38,6 +46,10 @@ variable "input_bucket_arn" {
   type = string
 }
 
+variable "dashboard_security_group_id" {
+  type = string
+}
+
 variable "db_host" {
   type = string
 }
@@ -62,6 +74,12 @@ variable "db_password_ssm_parameter_arn" {
   type = string
 }
 
+
+variable "api_port" {
+  type = number
+}
+
 variable "thread_count" {
   type = number
 }
+

variables.tf

@@ -248,6 +248,10 @@ variable "lawfulness_task_image" {
   type = string
 }
 
+variable "lawfulness_api_port" {
+  type = number
+}
+
 variable "lawfulness_thread_count" {
   type = number
 }