AITED-118: ci pipeline

dev.tfvars

@@ -17,8 +17,10 @@ terraform_s3_bucket_name      = "d-ew1-ted-ai-terraform"
 terraform_dynamodb_table_name = "d-ew1-ted-ai-terraform-locks"
 
 # TED AI project
-s3_input_bucket_name = "d-ew1-ted-ai-input"
-s3_data_bucket_name  = "d-ew1-ted-ai-experiments-data"
+s3_input_bucket_name   = "d-ew1-ted-ai-input"
+s3_data_bucket_name    = "d-ew1-ted-ai-experiments-data"
+s3_ml_data_bucket_name = "d-ew1-ted-ai-ml-data"
+s3_ml_models_bucket_name = "d-ew1-ted-ai-ml-models"
 
 ingestion_checkpoint_table = "d-ew1-ted-ai-ingestion-checkpoint"
 ingestion_tasks_table      = "d-ew1-ted-ai-ingestion-tasks"
@@ -26,9 +28,12 @@ ingestion_references_table = "d-ew1-ted-ai-ingestion-references"
 
 ingestion_tasks_queue_name = "d-ew1-ted-ai-ingestion-tasks-queue"
 
+sagemaker_classifiers_repository_name = "sagemaker-classifiers"
+
 # SSM path
 ssm_ingestion_checkpoint_id = "/dynamodb/ingestion_checkpoint/id"
 ssm_ingestion_tasks_id      = "/dynamodb/ingestion_tasks/id"
 ssm_ingestion_references_id = "/dynamodb/ingestion_references/id"
 ssm_buckets_input_id        = "/s3/input_bucket/id"
+ssm_buckets_ml_data_id      = "/s3/ml_data_bucket/id"
 ssm_ingestion_tasks_queue   = "/sqs/ingestion_tasks_queue/url"

main.tf

@@ -24,6 +24,8 @@ module "storage" {
   source                      = "./modules/storage"
   s3_input_bucket_name        = var.s3_input_bucket_name
   s3_data_bucket_name         = var.s3_data_bucket_name
+  s3_ml_data_bucket_name      = var.s3_ml_data_bucket_name
+  s3_ml_models_bucket_name    = var.s3_ml_models_bucket_name
   tags                        = var.tags
   ingestion_checkpoint_table  = var.ingestion_checkpoint_table
   ingestion_tasks_table       = var.ingestion_tasks_table
@@ -32,6 +34,7 @@ module "storage" {
   ssm_ingestion_tasks_id      = var.ssm_ingestion_tasks_id
   ssm_ingestion_references_id = var.ssm_ingestion_references_id
   ssm_buckets_input_id        = var.ssm_buckets_input_id
+  ssm_buckets_ml_data_id      = var.ssm_buckets_ml_data_id
 }
 
 module "queue" {
@@ -41,3 +44,9 @@ module "queue" {
   ssm_ingestion_tasks_queue  = var.ssm_ingestion_tasks_queue
 }
 
+module "classifiers" {
+  source = "./modules/classifiers"
+  sagemaker_classifiers_repository_name = var.sagemaker_classifiers_repository_name
+  tags = var.tags
+}
+

modules/classifiers/common.tf

+variable "tags" {
+  type = map(string)
+}
+
+variable "sagemaker_classifiers_repository_name" {
+  type = string
+}
\ No newline at end of file

modules/classifiers/ecr.tf

+resource "aws_ecr_repository" "sagemaker_classifiers" {
+  name                 = var.sagemaker_classifiers_repository_name
+  image_tag_mutability = "IMMUTABLE"
+
+  image_scanning_configuration {
+    scan_on_push = true
+  }
+
+  tags                       = var.tags
+}
+
+resource "aws_ecr_repository" "sagemaker_classifiers_ci" {
+  name                 = "${var.sagemaker_classifiers_repository_name}-ci"
+  image_tag_mutability = "IMMUTABLE"
+
+  image_scanning_configuration {
+    scan_on_push = true
+  }
+  tags                       = var.tags
+}
+
+resource "aws_ecr_lifecycle_policy" "sagemaker_classifiers_ci" {
+  repository = aws_ecr_repository.sagemaker_classifiers_ci.name
+
+  policy = <<EOF
+{
+    "rules": [
+        {
+            "rulePriority": 1,
+            "description": "Expire images older than 1 days",
+            "selection": {
+                "tagStatus": "any",
+                "countType": "sinceImagePushed",
+                "countUnit": "days",
+                "countNumber": 1
+            },
+            "action": {
+                "type": "expire"
+            }
+        }
+    ]
+}
+EOF
+}

modules/storage/s3.tf

@@ -85,3 +85,91 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "data_bucket" {
     }
   }
 }
+
+resource "aws_s3_bucket" "ml_data_bucket" {
+  bucket = var.s3_ml_data_bucket_name
+
+  # Prevents Terraform from destroying or replacing this object - a great safety mechanism
+  lifecycle {
+    prevent_destroy = true
+  }
+
+  tags = var.tags
+}
+
+resource "aws_s3_bucket_versioning" "ml_data_bucket" {
+  bucket = aws_s3_bucket.ml_data_bucket.id
+  versioning_configuration {
+    status = "Enabled"
+  }
+}
+
+resource "aws_s3_bucket_acl" "ml_data_bucket" {
+  bucket = aws_s3_bucket.ml_data_bucket.id
+  acl    = "private"
+}
+
+resource "aws_s3_bucket_public_access_block" "ml_data_bucket" {
+  bucket = aws_s3_bucket.ml_data_bucket.id
+
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+}
+
+resource "aws_s3_bucket_server_side_encryption_configuration" "ml_data_bucket" {
+  bucket = aws_s3_bucket.ml_data_bucket.id
+  rule {
+    apply_server_side_encryption_by_default {
+      sse_algorithm = "AES256"
+    }
+  }
+}
+
+resource "aws_ssm_parameter" "ml_data_bucket" {
+  name  = var.ssm_buckets_ml_data_id
+  type  = "String"
+  value = aws_s3_bucket.ml_data_bucket.id
+}
+
+resource "aws_s3_bucket" "ml_models_bucket" {
+  bucket = var.s3_ml_models_bucket_name
+
+  # Prevents Terraform from destroying or replacing this object - a great safety mechanism
+  lifecycle {
+    prevent_destroy = true
+  }
+
+  tags = var.tags
+}
+
+resource "aws_s3_bucket_versioning" "ml_models_bucket" {
+  bucket = aws_s3_bucket.ml_models_bucket.id
+  versioning_configuration {
+    status = "Enabled"
+  }
+}
+
+resource "aws_s3_bucket_acl" "ml_models_bucket" {
+  bucket = aws_s3_bucket.ml_models_bucket.id
+  acl    = "private"
+}
+
+resource "aws_s3_bucket_public_access_block" "ml_models_bucket" {
+  bucket = aws_s3_bucket.ml_models_bucket.id
+
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+}
+
+resource "aws_s3_bucket_server_side_encryption_configuration" "ml_models_bucket" {
+  bucket = aws_s3_bucket.ml_models_bucket.id
+  rule {
+    apply_server_side_encryption_by_default {
+      sse_algorithm = "AES256"
+    }
+  }
+}

modules/storage/s3_variables.tf

@@ -6,7 +6,19 @@ variable "s3_data_bucket_name" {
   type = string
 }
 
+variable "s3_ml_data_bucket_name" {
+  type = string
+}
+
+variable "s3_ml_models_bucket_name" {
+  type = string
+}
+
 variable "ssm_buckets_input_id" {
   type = string
 }
 
+variable "ssm_buckets_ml_data_id" {
+  type = string
+}
+

variables.tf

@@ -38,6 +38,14 @@ variable "s3_data_bucket_name" {
   type = string
 }
 
+variable "s3_ml_data_bucket_name" {
+  type = string
+}
+
+variable "s3_ml_models_bucket_name" {
+  type = string
+}
+
 variable "ingestion_checkpoint_table" {
   type = string
 }
@@ -54,6 +62,10 @@ variable "ingestion_tasks_queue_name" {
   type = string
 }
 
+variable "sagemaker_classifiers_repository_name" {
+  type = string
+}
+
 # SSM
 variable "ssm_ingestion_checkpoint_id" {
   type = string
@@ -71,6 +83,10 @@ variable "ssm_buckets_input_id" {
   type = string
 }
 
+variable "ssm_buckets_ml_data_id" {
+  type = string
+}
+
 variable "ssm_ingestion_tasks_queue" {
   type = string
 }
\ No newline at end of file