Code development platform for open source projects from the European Union institutions

Skip to content
Snippets Groups Projects
Commit 8fb3de20 authored by Vincent SIMONIN's avatar Vincent SIMONIN
Browse files

:twisted_rightwards_arrows: Merge branch 'rps_url_cleaning' into 'main'

:bug: Fixed source and target URL are not cleaned before storage

See merge request !57
parents f6ee6220 105b7f64
No related branches found
No related tags found
1 merge request!57🐛 Fixed source and target URL are not cleaned before storage
Pipeline #123291 passed
......@@ -3,16 +3,16 @@
tasks:
- name: Log into private registry
docker_login:
registry: "{{ lookup('ansible.builtin.env','CI_REGISTRY') }}"
username: "{{ lookup('ansible.builtin.env','CI_REGISTRY_USER') }}"
password: "{{ lookup('ansible.builtin.env','CI_REGISTRY_PASSWORD') }}"
community.docker.docker_login:
registry: "{{ lookup('ansible.builtin.env', 'CI_REGISTRY') }}"
username: "{{ lookup('ansible.builtin.env', 'CI_REGISTRY_USER') }}"
password: "{{ lookup('ansible.builtin.env', 'CI_REGISTRY_PASSWORD') }}"
reauthorize: true
- name: Remove image
community.docker.docker_image:
state: absent
name: "{{ lookup('ansible.builtin.env','CI_PROJECT_NAME') }}:{{ lookup('ansible.builtin.env','CI_COMMIT_SHORT_SHA') }}"
name: "{{ lookup('ansible.builtin.env', 'CI_PROJECT_NAME') }}:{{ lookup('ansible.builtin.env', 'CI_COMMIT_SHORT_SHA') }}"
force_absent: true
- name: Building image
......@@ -23,27 +23,27 @@
args:
http_proxy: "{{ lookup('ansible.builtin.env', 'HTTP_PROXY') }}"
https_proxy: "{{ lookup('ansible.builtin.env', 'HTTPS_PROXY') }}"
name: "{{ lookup('ansible.builtin.env','CI_PROJECT_NAME') }}:{{ lookup('ansible.builtin.env','CI_COMMIT_SHORT_SHA') }}"
repository: "{{ lookup('ansible.builtin.env','CI_REGISTRY_IMAGE') }}:{{ lookup('ansible.builtin.env','CI_COMMIT_SHORT_SHA') }}"
name: "{{ lookup('ansible.builtin.env', 'CI_PROJECT_NAME') }}:{{ lookup('ansible.builtin.env', 'CI_COMMIT_SHORT_SHA') }}"
repository: "{{ lookup('ansible.builtin.env', 'CI_REGISTRY_IMAGE') }}:{{ lookup('ansible.builtin.env', 'CI_COMMIT_SHORT_SHA') }}"
push: true
force_source: true
force_tag: true
source: build
- name: Get source image
set_fact:
ansible.builtin.set_fact:
source_image: "{{ lookup('ansible.builtin.file', '../Dockerfile') | regex_search('FROM (.*):(.*)') | regex_replace('^FROM\\s(.*)$', '\\1') }}"
- name: Remove local image
community.docker.docker_image:
state: absent
name: "{{ lookup('ansible.builtin.env','CI_PROJECT_NAME') }}:{{ lookup('ansible.builtin.env','CI_COMMIT_SHORT_SHA') }}"
name: "{{ lookup('ansible.builtin.env', 'CI_PROJECT_NAME') }}:{{ lookup('ansible.builtin.env', 'CI_COMMIT_SHORT_SHA') }}"
force_absent: true
- name: Remove local image
community.docker.docker_image:
state: absent
name: "{{ lookup('ansible.builtin.env','CI_REGISTRY_IMAGE') }}:{{ lookup('ansible.builtin.env','CI_COMMIT_SHORT_SHA') }}"
name: "{{ lookup('ansible.builtin.env', 'CI_REGISTRY_IMAGE') }}:{{ lookup('ansible.builtin.env', 'CI_COMMIT_SHORT_SHA') }}"
force_absent: true
- name: Remove source image
......@@ -56,6 +56,6 @@
community.docker.docker_login:
state: absent
- name: debug
debug:
msg: "{{ lookup('ansible.builtin.env','CI_REGISTRY_IMAGE') }}:{{ lookup('ansible.builtin.env','CI_COMMIT_SHORT_SHA') }}"
- name: Debug
ansible.builtin.debug:
msg: "{{ lookup('ansible.builtin.env', 'CI_REGISTRY_IMAGE') }}:{{ lookup('ansible.builtin.env', 'CI_COMMIT_SHORT_SHA') }}"
......@@ -3,21 +3,21 @@
tasks:
- name: Log into private registry
docker_login:
registry: "{{ lookup('ansible.builtin.env','CI_REGISTRY') }}"
username: "{{ lookup('ansible.builtin.env','CI_REGISTRY_USER') }}"
password: "{{ lookup('ansible.builtin.env','CI_REGISTRY_PASSWORD') }}"
community.docker.docker_login:
registry: "{{ lookup('ansible.builtin.env', 'CI_REGISTRY') }}"
username: "{{ lookup('ansible.builtin.env', 'CI_REGISTRY_USER') }}"
password: "{{ lookup('ansible.builtin.env', 'CI_REGISTRY_PASSWORD') }}"
reauthorize: true
- name: Get image to deliver
community.docker.docker_image:
name: "{{ lookup('ansible.builtin.env','CI_REGISTRY_IMAGE') }}:{{ lookup('ansible.builtin.env','CI_COMMIT_SHORT_SHA') }}"
name: "{{ lookup('ansible.builtin.env', 'CI_REGISTRY_IMAGE') }}:{{ lookup('ansible.builtin.env', 'CI_COMMIT_SHORT_SHA') }}"
source: pull
- name: Push tag image
community.docker.docker_image:
name: "{{ lookup('ansible.builtin.env','CI_REGISTRY_IMAGE') }}:{{ lookup('ansible.builtin.env','CI_COMMIT_SHORT_SHA') }}"
repository: "{{ lookup('ansible.builtin.env','CI_REGISTRY_IMAGE') }}:{{ lookup('ansible.builtin.env','IMAGE_TAG') }}"
name: "{{ lookup('ansible.builtin.env', 'CI_REGISTRY_IMAGE') }}:{{ lookup('ansible.builtin.env', 'CI_COMMIT_SHORT_SHA') }}"
repository: "{{ lookup('ansible.builtin.env', 'CI_REGISTRY_IMAGE') }}:{{ lookup('ansible.builtin.env', 'IMAGE_TAG') }}"
push: true
force_tag: true
source: local
......@@ -25,13 +25,13 @@
- name: Remove local image
community.docker.docker_image:
state: absent
name: "{{ lookup('ansible.builtin.env','CI_REGISTRY_IMAGE') }}:{{ lookup('ansible.builtin.env','CI_COMMIT_SHORT_SHA') }}"
name: "{{ lookup('ansible.builtin.env', 'CI_REGISTRY_IMAGE') }}:{{ lookup('ansible.builtin.env', 'CI_COMMIT_SHORT_SHA') }}"
force_absent: true
- name: Remove local image
community.docker.docker_image:
state: absent
name: "{{ lookup('ansible.builtin.env','CI_REGISTRY_IMAGE') }}:{{ lookup('ansible.builtin.env','IMAGE_TAG') }}"
name: "{{ lookup('ansible.builtin.env', 'CI_REGISTRY_IMAGE') }}:{{ lookup('ansible.builtin.env', 'IMAGE_TAG') }}"
force_absent: true
- name: Log out of registry
......
......@@ -4,43 +4,48 @@
tasks:
- name: Create netbox directory
ansible.builtin.file:
path: "/home/debian/netbox/{{ lookup('ansible.builtin.env','CI_PIPELINE_ID') }}"
path: "/home/debian/netbox/{{ lookup('ansible.builtin.env', 'CI_PIPELINE_ID') }}"
state: directory
mode: "0755"
- name: Copy docker-compose file
ansible.builtin.copy:
src: "../docker-compose.yml"
dest: "/home/debian/netbox/{{ lookup('ansible.builtin.env','CI_PIPELINE_ID') }}/docker-compose.yml"
dest: "/home/debian/netbox/{{ lookup('ansible.builtin.env', 'CI_PIPELINE_ID') }}/docker-compose.yml"
mode: "0644"
- name: Copy testing docker-compose file
ansible.builtin.copy:
src: "../docker-compose.test.yml"
dest: "/home/debian/netbox/{{ lookup('ansible.builtin.env','CI_PIPELINE_ID') }}/docker-compose.override.yml"
dest: "/home/debian/netbox/{{ lookup('ansible.builtin.env', 'CI_PIPELINE_ID') }}/docker-compose.override.yml"
mode: "0644"
- name: Copy testing env variables
ansible.builtin.copy:
src: "../env"
dest: "/home/debian/netbox/{{ lookup('ansible.builtin.env','CI_PIPELINE_ID') }}/"
dest: "/home/debian/netbox/{{ lookup('ansible.builtin.env', 'CI_PIPELINE_ID') }}/"
mode: "0755"
- name: Create .env file
ansible.builtin.copy:
dest: "/home/debian/netbox/{{ lookup('ansible.builtin.env','CI_PIPELINE_ID') }}/.env"
dest: "/home/debian/netbox/{{ lookup('ansible.builtin.env', 'CI_PIPELINE_ID') }}/.env"
content: |
TAG={{ lookup('ansible.builtin.env','CI_COMMIT_SHORT_SHA') }}
HOSTNAME={{ lookup('ansible.builtin.env','HOSTNAME') }}
TAG={{ lookup('ansible.builtin.env', 'CI_COMMIT_SHORT_SHA') }}
HOSTNAME={{ lookup('ansible.builtin.env', 'HOSTNAME') }}
mode: "0644"
- name: Run `docker-compose up`
community.docker.docker_compose:
project_src: "/home/debian/netbox/{{ lookup('ansible.builtin.env','CI_PIPELINE_ID') }}/"
project_src: "/home/debian/netbox/{{ lookup('ansible.builtin.env', 'CI_PIPELINE_ID') }}/"
state: present
pull: true
- name: Wait until the metrics are available
ansible.builtin.uri:
url: "http://{{ lookup('ansible.builtin.env','CI_COMMIT_SHORT_SHA') }}.{{ lookup('ansible.builtin.env', 'HOSTNAME') }}/metrics"
url: "http://{{ lookup('ansible.builtin.env', 'CI_COMMIT_SHORT_SHA') }}.{{ lookup('ansible.builtin.env', 'HOSTNAME') }}/metrics"
status_code: 200
register: curl_output
until: curl_output.status == 200
until: curl_output.status == 200
retries: 60
delay: 5
delegate_to: localhost
......
......@@ -4,7 +4,7 @@
tasks:
- name: Run `docker-compose down`
community.docker.docker_compose:
project_src: "/home/debian/netbox/{{ lookup('ansible.builtin.env','CI_PIPELINE_ID') }}/"
project_src: "/home/debian/netbox/{{ lookup('ansible.builtin.env', 'CI_PIPELINE_ID') }}/"
state: absent
remove_volumes: true
remove_images: "all"
......@@ -12,5 +12,5 @@
- name: Remove netbox directory
ansible.builtin.file:
path: "/home/debian/netbox/{{ lookup('ansible.builtin.env','CI_PIPELINE_ID') }}"
path: "/home/debian/netbox/{{ lookup('ansible.builtin.env', 'CI_PIPELINE_ID') }}"
state: absent
......@@ -3,10 +3,10 @@
tasks:
- name: Log into private registry
docker_login:
registry: "{{ lookup('ansible.builtin.env','CI_REGISTRY') }}"
username: "{{ lookup('ansible.builtin.env','CI_REGISTRY_USER') }}"
password: "{{ lookup('ansible.builtin.env','CI_REGISTRY_PASSWORD') }}"
community.docker.docker_login:
registry: "{{ lookup('ansible.builtin.env', 'CI_REGISTRY') }}"
username: "{{ lookup('ansible.builtin.env', 'CI_REGISTRY_USER') }}"
password: "{{ lookup('ansible.builtin.env', 'CI_REGISTRY_PASSWORD') }}"
reauthorize: true
- name: Start up services
......
......@@ -30,4 +30,3 @@ volumes:
driver: local
netbox-scripts-files:
driver: local
......@@ -8,8 +8,12 @@ services:
start_period: 2s
retries: 30
env_file: env/netbox.env
environment:
- DB_HOST=postgres_${TAG}
- REDIS_CACHE_HOST=redis_cache_${TAG}
- REDIS_HOST=redis_${TAG}
labels:
- "traefik.http.routers.netbox.rule=Host(`${TAG}.${HOSTNAME}`)"
- "traefik.http.routers.netbox-${TAG}.rule=Host(`${TAG}.netbox.ntx.lu`)"
networks:
- traefik
logging:
......@@ -41,6 +45,7 @@ services:
syslog-format: "rfc5424"
tag: "netbox-housekeeping"
postgres:
container_name: postgres_${TAG}
env_file: env/postgres.env
networks:
- traefik
......@@ -51,6 +56,7 @@ services:
syslog-format: "rfc5424"
tag: "netbox-postgres"
redis:
container_name: redis_${TAG}
env_file: env/redis.env
networks:
- traefik
......@@ -61,6 +67,7 @@ services:
syslog-format: "rfc5424"
tag: "netbox-redis"
redis-cache:
container_name: redis_cache_${TAG}
env_file: env/redis-cache.env
networks:
- traefik
......
CORS_ORIGIN_ALLOW_ALL=True
DB_HOST=postgres
DB_NAME=netbox
DB_PASSWORD=J5brHrAXFLQSif0K
DB_USER=netbox
......@@ -20,12 +19,10 @@ HOUSEKEEPING_INTERVAL=86400
MEDIA_ROOT=/opt/netbox/netbox/media
METRICS_ENABLED=true
REDIS_CACHE_DATABASE=1
REDIS_CACHE_HOST=redis-cache
REDIS_CACHE_INSECURE_SKIP_TLS_VERIFY=false
REDIS_CACHE_PASSWORD=t4Ph722qJ5QHeQ1qfu36
REDIS_CACHE_SSL=false
REDIS_DATABASE=0
REDIS_HOST=redis
REDIS_INSECURE_SKIP_TLS_VERIFY=false
REDIS_PASSWORD=H733Kdjndks81
REDIS_SSL=false
......
......@@ -2,7 +2,7 @@
from rest_framework import serializers
from netbox.api.serializers import NetBoxModelSerializer, WritableNestedSerializer
from ..models import Mapping, HttpHeader, SamlConfig
from ..models import Mapping, HttpHeader, SamlConfig, clean_url
class NestedMappingSerializer(WritableNestedSerializer):
......@@ -132,3 +132,23 @@ class MappingSerializer(NetBoxModelSerializer):
"http_headers",
"saml_config",
)
def create(self, validated_data):
"""Be sure that URL is cleaned"""
instance = super().create(validated_data)
instance.source = clean_url(instance.source)
instance.target = clean_url(instance.target)
instance.save()
return instance
def update(self, instance, validated_data):
"""Be sure that URL is cleaned"""
validated_data["source"] = clean_url(validated_data["source"])
validated_data["target"] = clean_url(validated_data["target"])
return super().update(instance, validated_data)
"""Models definitions"""
from typing import Any
from urllib.parse import urlparse
from django.core.exceptions import ValidationError
from django.conf import settings
from django.db import models
from django.db.models import Model
from django.urls import reverse
from django.core.validators import URLValidator, MaxValueValidator, MinValueValidator
from django.contrib.postgres.fields.array import ArrayField
......@@ -13,6 +16,33 @@ URL_MAX_SIZE = 2000
DEFAULT_SORRY_PAGE = settings.PLUGINS_CONFIG["netbox_rps_plugin"]["default_sorry_page"]
def clean_url(raw_url):
"""Clean an URL"""
o = urlparse(raw_url)
credential = ":".join(
tuple(filter(lambda item: item is not None, (o.username, o.password)))
)
hostname = o.hostname if o.port is None else ":".join((o.hostname, str(o.port)))
return (
o._replace(netloc=hostname)
if len(credential) == 0
else o._replace(netloc="@".join((credential, hostname)))
).geturl()
class FilteredURLField(models.URLField):
"""URLField definition class"""
def clean(self, value: Any, model_instance: Model | None) -> Any:
"""Clean Field value"""
return clean_url(super().clean(value, model_instance))
class AuthenticationChoices(ChoiceSet):
"""Authentication choices definition class"""
......@@ -60,18 +90,18 @@ def default_protocol():
class Mapping(NetBoxModel):
"""Mapping definition class"""
source = models.CharField(
source = FilteredURLField(
max_length=URL_MAX_SIZE,
blank=False,
verbose_name="Source",
validators=[URLValidator(message="It must be a url")],
validators=[URLValidator(schemes=["http", "https"])],
unique=True,
)
target = models.CharField(
target = FilteredURLField(
max_length=URL_MAX_SIZE,
blank=False,
verbose_name="Target",
validators=[URLValidator(message="It must be a url")],
validators=[URLValidator(schemes=["http", "https"])],
)
authentication = models.CharField(
max_length=30,
......@@ -80,11 +110,11 @@ class Mapping(NetBoxModel):
blank=False,
verbose_name="Auth",
)
testingpage = models.CharField(
testingpage = models.URLField(
max_length=URL_MAX_SIZE,
blank=True,
null=True,
validators=[URLValidator(message="It must be a url")],
validators=[URLValidator(schemes=["http", "https"])],
)
webdav = models.BooleanField(
default=False,
......@@ -104,11 +134,11 @@ class Mapping(NetBoxModel):
client_max_body_size = models.IntegerField(
default=1, validators=[MinValueValidator(1), MaxValueValidator(255)]
)
sorry_page = models.CharField(
sorry_page = models.URLField(
max_length=URL_MAX_SIZE,
blank=False,
verbose_name="Sorry Page",
validators=[URLValidator(message="It must be a url")],
validators=[URLValidator(schemes=["http", "https"])],
default=DEFAULT_SORRY_PAGE,
)
extra_protocols = ArrayField(
......
......@@ -56,8 +56,8 @@ class TestMappingUnique(Base):
response = requests.post(
url=f"http://{HOST}:{PORT}/api/plugins/rps/mapping/",
json={
"source": "https://truc8.com/api",
"target": "https://truc8.com/api",
"source": "https://truc9.com/api",
"target": "https://truc9.com/api",
"authentication": "none",
"testingpage": None,
},
......@@ -71,6 +71,152 @@ class TestMappingUnique(Base):
b'{"target":["Target URL cannot be equal to source URL."]}',
)
def test_that_mapping_is_case_sensitive_unique(self) -> None:
"""Test that mapping is case sensitive unique"""
response = requests.post(
url=f"http://{HOST}:{PORT}/api/plugins/rps/mapping/",
json={
"source": "https://truc10.com/api",
"target": "http://10.10.10.10:1888/api",
"authentication": "none",
"testingpage": None,
},
headers={"Authorization": f"Token {API_KEY}"},
timeout=5,
)
self.assertEqual(response.status_code, 201)
self.mapping_id = json.loads(response.content)["id"]
response = requests.post(
url=f"http://{HOST}:{PORT}/api/plugins/rps/mapping/",
json={
"source": "HTTPS://truc10.com/api",
"target": "http://10.10.10.10:1888/api",
"authentication": "none",
"testingpage": None,
},
headers={"Authorization": f"Token {API_KEY}"},
timeout=5,
)
self.assertEqual(response.status_code, 400)
self.assertEqual(
response.content, b'{"source":["Mapping with this Source already exists."]}'
)
response = requests.post(
url=f"http://{HOST}:{PORT}/api/plugins/rps/mapping/",
json={
"source": "https://TRUC10.COM/api",
"target": "http://10.10.10.10:1888/api",
"authentication": "none",
"testingpage": None,
},
headers={"Authorization": f"Token {API_KEY}"},
timeout=5,
)
self.assertEqual(response.status_code, 400)
self.assertEqual(
response.content, b'{"source":["Mapping with this Source already exists."]}'
)
response = requests.post(
url=f"http://{HOST}:{PORT}/api/plugins/rps/mapping/",
json={
"source": "HTTPS://TRUC10.com/API",
"target": "HTTP://Toto.10.com:1888/aPi",
"authentication": "none",
"testingpage": None,
},
headers={"Authorization": f"Token {API_KEY}"},
timeout=5,
)
self.assertEqual(response.status_code, 201)
content = json.loads(response.content)
self.assertEqual(content["source"], "https://truc10.com/API")
self.assertEqual(content["target"], "http://toto.10.com:1888/aPi")
response = requests.get(
url=f"http://{HOST}:{PORT}/api/plugins/rps/mapping/{content['id']}/",
headers={"Authorization": f"Token {API_KEY}"},
timeout=5,
)
self.assertEqual(response.status_code, 200)
content = json.loads(response.content)
self.assertEqual(content["source"], "https://truc10.com/API")
self.assertEqual(content["target"], "http://toto.10.com:1888/aPi")
requests.delete(
url=f"http://{HOST}:{PORT}/api/plugins/rps/mapping/",
json=[{"id": content["id"]}],
headers={"Authorization": f"Token {API_KEY}"},
timeout=5,
)
def test_that_mapping_update_is_case_sensitive_unique(self) -> None:
"""Test that mapping update is case sensitive unique"""
response = requests.post(
url=f"http://{HOST}:{PORT}/api/plugins/rps/mapping/",
json={
"source": "HTTPS://TRUC11.com/API",
"target": "HTTP://Toto.11.com:1888/aPi",
"authentication": "none",
"testingpage": None,
},
headers={"Authorization": f"Token {API_KEY}"},
timeout=5,
)
self.assertEqual(response.status_code, 201)
content = json.loads(response.content)
response = requests.patch(
url=f"http://{HOST}:{PORT}/api/plugins/rps/mapping/{content['id']}/",
json={
"source": "HTTPS://MUCHE11.com/API",
"target": "HTTP://Titi.11.com:1888/aPi",
},
headers={"Authorization": f"Token {API_KEY}"},
timeout=5,
)
self.assertEqual(response.status_code, 200)
content = json.loads(response.content)
self.assertEqual(content["source"], "https://muche11.com/API")
self.assertEqual(content["target"], "http://titi.11.com:1888/aPi")
response = requests.get(
url=f"http://{HOST}:{PORT}/api/plugins/rps/mapping/{content['id']}/",
headers={"Authorization": f"Token {API_KEY}"},
timeout=5,
)
self.assertEqual(response.status_code, 200)
content = json.loads(response.content)
self.assertEqual(content["source"], "https://muche11.com/API")
self.assertEqual(content["target"], "http://titi.11.com:1888/aPi")
requests.delete(
url=f"http://{HOST}:{PORT}/api/plugins/rps/mapping/",
json=[{"id": content["id"]}],
headers={"Authorization": f"Token {API_KEY}"},
timeout=5,
)
if __name__ == "__main__":
unittest.main()
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment