diff --git a/package-lock.json b/package-lock.json index 1cc3347c99a7053445b53c612eb905c67c3153ec..37d077ed21999bec4ba4c82553d339708e7d6638 100644 --- a/package-lock.json +++ b/package-lock.json @@ -283,26 +283,26 @@ } }, "node_modules/@babel/helpers": { - "version": "7.26.9", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.26.9.tgz", - "integrity": "sha512-Mz/4+y8udxBKdmzt/UjPACs4G3j5SshJJEFFKxlCGPydG4JAHXxjWjAwjd09tf6oINvl1VfMJo+nB7H2YKQ0dA==", + "version": "7.27.0", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.27.0.tgz", + "integrity": "sha512-U5eyP/CTFPuNE3qk+WZMxFkp/4zUzdceQlfzf7DdGdhp+Fezd7HD+i8Y24ZuTMKX3wQBld449jijbGq6OdGNQg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/template": "^7.26.9", - "@babel/types": "^7.26.9" + "@babel/template": "^7.27.0", + "@babel/types": "^7.27.0" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/parser": { - "version": "7.26.9", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.26.9.tgz", - "integrity": "sha512-81NWa1njQblgZbQHxWHpxxCzNsa3ZwvFqpUg7P+NNUU6f3UU2jBEg4OlF/J6rl8+PQGh1q6/zWScd001YwcA5A==", + "version": "7.27.0", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.27.0.tgz", + "integrity": "sha512-iaepho73/2Pz7w2eMS0Q5f83+0RKI7i4xmiYeBmDzfRVbQtTOG7Ts0S4HzJVsTMGI9keU8rNfuZr8DKfSt7Yyg==", "license": "MIT", "dependencies": { - "@babel/types": "^7.26.9" + "@babel/types": "^7.27.0" }, "bin": { "parser": "bin/babel-parser.js" @@ -551,15 +551,15 @@ } }, "node_modules/@babel/template": { - "version": "7.26.9", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.26.9.tgz", - "integrity": "sha512-qyRplbeIpNZhmzOysF/wFMuP9sctmh2cFzRAZOn1YapxBsE1i9bJIY586R/WBLfLcmcBlM8ROBiQURnnNy+zfA==", + "version": "7.27.0", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.27.0.tgz", + "integrity": "sha512-2ncevenBqXI6qRMukPlXwHKHchC7RyMuu4xv5JBXRfOGVcTy1mXCD12qrp7Jsoxll1EV3+9sE4GugBVRjT2jFA==", "dev": true, "license": "MIT", "dependencies": { "@babel/code-frame": "^7.26.2", - "@babel/parser": "^7.26.9", - "@babel/types": "^7.26.9" + "@babel/parser": "^7.27.0", + "@babel/types": "^7.27.0" }, "engines": { "node": ">=6.9.0" @@ -610,9 +610,9 @@ "license": "MIT" }, "node_modules/@babel/types": { - "version": "7.26.9", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.9.tgz", - "integrity": "sha512-Y3IR1cRnOxOCDvMmNiym7XpXQ93iGDDPHx+Zj+NM+rg0fBaShfQLkg+hKPaZCEvg5N/LeCo4+Rj/i3FuJsIQaw==", + "version": "7.27.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.27.0.tgz", + "integrity": "sha512-H45s8fVLYjbhFH62dIJ3WtmJ6RSPt/3DRO0ZcT2SUiYiQyz3BLVb9ADEnLl91m74aQPS3AzzeajZHYOalWe3bg==", "license": "MIT", "dependencies": { "@babel/helper-string-parser": "^7.25.9", diff --git a/src/assets/template-dnt-comparison.pug b/src/assets/template-dnt-comparison.pug new file mode 100644 index 0000000000000000000000000000000000000000..d2d87141f78c268ee456238dc8f01cd58ead7193 --- /dev/null +++ b/src/assets/template-dnt-comparison.pug @@ -0,0 +1,946 @@ +doctype html +html(xmlns="http://www.w3.org/1999/xhtml", xml:lang="en", lang="en") + head + meta(charset="utf-8") + meta(name="generator", content="website-evidence-collector") + meta( + name="viewport", + content="width=device-width, initial-scale=1.0, user-scalable=yes" + ) + title #{ title } (#{ uri_ins }) + base(target="_blank")/ + style(type="text/css") !{ inlineCSS } + style(type="text/css"). + /* pandoc */ + code { + white-space: pre-wrap; + } + span.smallcaps { + font-variant: small-caps; + } + span.underline { + text-decoration: underline; + } + div.column { + display: inline-block; + vertical-align: top; + width: 50%; + } + + /* github */ + .markdown-body > article { + box-sizing: border-box; + min-width: 200px; + max-width: 980px; + margin: 0 auto; + padding: 45px; + } + + .markdown-body table { + width: inherit !important; + } + + @media (max-width: 767px) { + .markdown-body > article { + padding: 15px; + } + } + + /* custom */ + .markdown-body { + counter-reset: h1counter; + } + + h1:before { + content: counter(h1counter) "\0000a0\0000a0"; + counter-increment: h1counter; + } + h1 { + counter-reset: h2counter; + } + + h2:before { + content: counter(h1counter) "." counter(h2counter) "\0000a0\0000a0"; + counter-increment: h2counter; + } + h2 { + counter-reset: h3counter; + } + + h3:before { + content: counter(h1counter) "." counter(h2counter) "." counter(h3counter) "\0000a0\0000a0"; + counter-increment: h3counter; + } + + h1.nocount:before, + h2.nocount:before, + h3.nocount:before { + content: none; + counter-increment: none; + } + + /* annex */ + h1[id^="app"]:before { + content: none; + } + h1[id^="app"] { + counter-reset: h2counter; + } + + h2[id^="app"]:before { + content: counter(h2counter, upper-alpha) "\0000a0\0000a0"; + } + + #logo { + width: 40%; + float: right; + background-color: var(--color-canvas-default); + } + + .notrunc { + white-space: nowrap; + } + + .trunc { + white-space: nowrap; + text-overflow: ellipsis; + overflow: hidden; + max-width: 1px; + } + + .markdown-body table td.code { + padding: 0px; + } + + .markdown-body table td.code pre { + margin: 0px; + } + + td.highlighted, + td.highlighted pre, + li.highlighted a { + background-color: red; + color: white; + } + + @media print { + .markdown-body h1, + h2, + h3, + h4, + h5, + h6 { + break-after: avoid-page; + } + + .screen-only { + display: none; + } + } + + .unnumbered::before { + content: none !important; + counter-increment: none; + } + + body.markdown-body: article + header#title-block-header + #logo + include /wec_logo.svg + h1.title.unnumbered= title + h2.subtitle.unnumbered: a(href=uri_ins)= uri_ins + + h1(id="sec:evidence-collection-organisation") Evidence collection organisation + + table + colgroup + col(style="width: 50%") + col(style="width: 50%") + tbody + tr + td Target web service + td: code= uri_ins + tr + td Automated evidence collection start time + td= new Date(start_time).toLocaleString("en-GB") + tr + td Automated evidence collection end time + td= new Date(end_time).toLocaleString("en-GB") + tr + td Software version + td= script.version.commit || script.version.npm + + h1(id="sec:automated-evidence-collection") Automated evidence collection + + p The automated evidence collection is carried out using the tool #[a(href="https://edps.europa.eu/press-publications/edps-inspection-software_en") website evidence collector], WEC (also #[a(href="https://code.europa.eu/EDPS/website-evidence-collector") on Code Europa EU]) in version #{ script.version.commit || script.version.npm } on the platform #{ browser.platform.name } in version #{ browser.platform.version }. The tool employs the browser #{ browser.name } in version #{ browser.version } for browsing the website. + + p The evidence collection tool simulates a browsing session of the web service, capturing traffic between the browser and the Internet, along with any persistent data stored in the browser. While browsing, the tool gathers evidence and performs a number of checks. + + p It captures screenshots from the browser to identify potential cookie banners. It also tests HTTPS/SSL usage to determine whether the website enforces a secure connection. Then, the evidence collection tool scans the first web page for links to common social media and collaboration platforms, gathering data on the overall use of potentially privacy-intrusive third-party web services. + + p The recorded traffic between the browser, the target web service, and involved third-party web services, as well as the browser’s persistent storage, will be analysed in a #[span.citation(data-cites="sec:traffic-and-persistent-data-analysis"): a(href="#traffic-and-persistent-data-analysis") subsequent section]. + + p Generally, the tool browses a random subset of the target web service pages starting from the initial web page. However, the browsing can also include a set of predefined web pages. The exhaustive list of browsed web pages for this specific evidence collection is given in #[span.citation(data-cites="app:history"): a(href="#app:history") the Annex: Browsing history]. + + h2(id="sec:webpage-visit") Web page visit + + p + | On #{ new Date(start_time).toLocaleString("en-GB") }, the evidence collection tool navigated the browser to #[a(href=uri_ins)= uri_ins]. The final location after potential redirects was #[a(href=uri_dest)= uri_dest]. + if script.config.screenshots + | + | The evidence collection tool took two screenshots #[span.citation(data-cites="fig:screenshot-top") to cover the top of the web page] and #[span.citation(data-cites="fig:screenshot-bottom") the bottom]. + + if script.config.screenshots + figure + img(id="fig:screenshot-top", + src=`data:image/png;base64,${screenshots.screenshot_top}`, + alt="Web page top screenshot", + style="width: 100%" + ) + figcaption Web page top screenshot + + figure + img(id="fig:screenshot-bottom", + src=`data:image/png;base64,${screenshots.screenshot_bottom}`, + alt="Web page bottom screenshot", + style="width: 100%" + ) + figcaption Web page bottom screenshot + + h2(id="sec:use-of-httpsssl") Use of HTTPS/SSL + + p HTTP (Hypertext Transfer Protocol) is a communication standard that transmits data between a website and a user’s browser in an unencrypted format, making it vulnerable to interception and eavesdropping. In contrast, HTTPS (Hypertext Transfer Protocol Secure) extends HTTP by adding an extra layer of security through encryption, which protects the confidentiality and integrity of the data exchanged between a website and a user’s browser. + + p The evidence collection tool assessed the behaviour of #{ host } with respect to the use of HTTPS. + + table.use-of-httpsssl + colgroup + col(style="width: 50%") + col(style="width: 50%") + tbody + tr + td Allows connection with HTTPS + td= secure_connection.https_support + tr + td HTTP redirect to HTTPS + td= secure_connection.https_redirect + if secure_connection.redirects + tr + td HTTP redirect location + td: ul + each redirect in secure_connection.redirects + li + a(href=redirect)= redirect + if secure_connection.http_error + tr + td Error when connecting with HTTP + td= secure_connection.http_error + if secure_connection.https_error + tr + td Error when connecting with HTTPS + td= secure_connection.https_error + + if testSSL && testSSL.scanResult[0] + - var results = testSSL.scanResult[0]; + + - + sortSeverity = function(a,b) { + var severityToNumber = { + CRITCAL: 0, + HIGH: 1, + MEDIUM: 2, + LOW: 3, + OK: 4, + INFO: 5, + }; + + return severityToNumber[a.severity]-severityToNumber[b.severity]; + } + + p The software TestSSL from #[a(href="https://testssl.sh") https://testssl.sh] inspected the HTTPS configuration of the web service host #{ results.targetHost }. It classifies detected vulnerabilities by their level of severity #[em low], #[em medium], #[em high], or #[em critical]. The severity ratings are automatically computed by the TestSSL software without considering the security requirements of the individual website. They do not reflect the opinions or views of the website evidence collector's authors. Details of the findings are listed in #[span.citation(data-cites="app:testssl"): a(target="_parent", href="#app:testssl") the Annex: TestSSL scan]. + + table.testssl-summary + thead + tr + th HTTPS/SSL vulnerabilities per severity + th.notrunc Freq. + tbody + - var vulnerabilitiesBySeverity = groupBy(results.vulnerabilities, "severity"); + tr + td Critical + td.notrunc= vulnerabilitiesBySeverity["CRITCAL"] ? vulnerabilitiesBySeverity["CRITCAL"].length : 0 + tr + td High + td.notrunc= vulnerabilitiesBySeverity["HIGH"] ? vulnerabilitiesBySeverity["HIGH"].length : 0 + tr + td Medium + td.notrunc= vulnerabilitiesBySeverity["MEDIUM"] ? vulnerabilitiesBySeverity["MEDIUM"].length : 0 + tr + td Low + td.notrunc= vulnerabilitiesBySeverity["LOW"] ? vulnerabilitiesBySeverity["LOW"].length : 0 + + h2(id="sec:use-of-csp") Use of content security policies (CSPs) + + p Upon a browser's request for a web page, websites can specify a whitelist of mechanisms, domains, and subdomains in the #[a(href="https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP") Content Security Policy] (CSP) metadata sent along with the requested page. Browsers must respect this whitelist when embedding components such as styles, fonts, beacons, videos, and maps. + + if hosts.contentSecurityPolicy.firstParty.length === 0 && hosts.contentSecurityPolicy.thirdParty.length === 0 + p No CSP metadata was found. Consequently, no restrictions apply. + + else + if hosts.contentSecurityPolicy.firstParty.length > 0 + ol + // check if host looks like a host (instead of e.g blob: data: etc.) + each host in hosts.contentSecurityPolicy.firstParty + if host.match(/[^\.]+\.[^\.]+/) + li: a(href=`http://${host}`)= host + else + li= host + + p The website has whitelisted #{ hosts.contentSecurityPolicy.firstParty.length } first-party domains and mechanisms. + else + p No CSP metadata related to first-party URLs was found. + + if hosts.contentSecurityPolicy.thirdParty.length > 0 + h4 Third-party content security policy hosts + + ol + each host in hosts.contentSecurityPolicy.thirdParty + li: a(href=`http://${host}`)= host + + p The website has whitelisted #{ hosts.contentSecurityPolicy.thirdParty.length } distinct third-party host(s). + else + p No third-party content security policy hosts were whitelisted. + + h2(id="sec:use-of-social-media") Use of social media and collaboration platforms + + p The website evidence collection tool found links from #[a(href=uri_dest)= uri_dest] to the following common social media and collaboration platforms. + + if links.social.length > 0 + table.use-of-social-media-and-collaboration-platforms( + style="width: 100%" + ) + colgroup + col(style="width: 100%") + col(style="width: 100%") + thead + tr + th Link URL + th Link caption + tbody + each social in links.social + tr + td.trunc: a(href=social.href)= social.href + td.notrunc= social.inner_text + else + p No corresponding links were found. + + h2#traffic-and-persistent-data-analysis Traffic and persistent data analysis + + p First, the browser visited #[a(href=uri_dest)= uri_dest]. The evidence collection navigated and collected evidence from #{ browsing_history.length > 1 ? browsing_history.length - 1 : "no" } additional web service page(s). + + p The web page(s) were browsed consecutively between #{ new Date(start_time).toLocaleString("en-GB") } and #{ new Date(end_time).toLocaleString("en-GB") }. + + p During the browsing, the HTTP Header #[a(href="https://en.wikipedia.org/wiki/Do_Not_Track") Do Not Track] was #{ browser.extra_headers.dnt ? 'set' : 'not set' }. + + p For the subsequent analysis, the following URLs (hosts with their paths) were defined as first-party: + + ol + each uri in uri_refs + li: a(href=uri)= uri.replace(/(^\w+:|^)\/\//, "") + + h3(id="sec:traffic-analysis") Traffic analysis + + p In the case of a visit to a very simple web page with a given URL (e.g. http://example.com/home.html), the browser sends a #[em request] to the web server configured for the domain specified in the URL (e.g. example.com). The web server, also called the #[em host], then sends a #[em response] in the form of, e.g. an HTML file (e.g. the home.html file), which the browser downloads and displays. Most web pages nowadays are more complex and include content such as images, videos, and fonts, or embed elements like maps, tweets, and comments. To assemble and show the whole web page, the browser sends further requests to the same host (#[em first-party]) or even different hosts (potentially #[em third-party]) to download the required content. A web page is often composed of dozens of elements, and due to the complexity of website architecture, website administrators are often not fully aware of all third parties involved in the functioning of their websites. + + p The evidence collection tool extracted lists of distinct first- and third-party hosts from the browser requests recorded in each browsing session (with DNT signal set and without). These lists are presented below and aim to help by providing a comprehensive overview of all the hosts from which the browser requests elements. Note that subdomains (e.g. admin.example.com) of first-party domains (example.com) are, by default, considered third-party domains, whereas all URLs in the path (e.g. example.com/anysubpage) are treated as first-party by the automated evidence collection tool. More information about hosts and the distinction between first-party and third-party can be found in the glossary in #[span.citation(data-cites="app:glossary"): a(href="#app:glossary") the Annex: Glossary]. + + p A number of techniques allow hosts to track browsing behaviour. A first-party host may instruct the browser to send requests solely for the purpose of providing information embedded in the request (e.g. cookies) to a given first-party or third-party host. These requests are often responded to with an empty file or a 1x1 pixel image. Such files requested for tracking purposes are commonly referred to as #[em web beacons]. + + p The evidence collection tool compares all requests against signature lists compiled to detect potential web beacons or annoyances such as in-page pop-ups. Positive matches with the lists #[a(href="https://easylist.to/#easyprivacy") EasyPrivacy] (#[code easyprivacy.txt]) and #[a(href="https://easylist.to/#fanboy-s-annoyance-list") Fanboy’s Annoyance] (#[code fanboy-annoyance.txt]) from #[a(href="https://easylist.to") https://easylist.to] are presented in #[span.citation(data-cites="app:annex-beacons"): a(href="#app:annex-beacons") the Annex: All potential web beacons]. The list of #[em web beacon hosts] contains hosts of those requests that match the signature list EasyPrivacy. Note that the result may include false positives and may be incomplete due to inaccurate, outdated or incomplete signature lists. + + p #[em Cookies] are small text files stored on a user’s browser that allow websites to track and store information about the user’s interactions. However, they are limited in capacity and are transmitted with every HTTP request. #[em Local storage objects], on the other hand, offer a more modern method for websites to store larger amounts of data locally on a user’s browser, with better control over data access and expiration. Both cookies and local storage objects can be used for tracking purposes. + + p Eventually, the evidence collection tool logged all identified web forms that potentially transmit web form data using an unencrypted connection. + + h4 First-party hosts + + ol + each host in hosts.requests.firstParty + li: a(href=`http://${host}`)= host + + p Requests have been made to #{ hosts.requests.firstParty.length } distinct first-party host(s). + + h4 Third-party hosts + + ol + each host in hosts.requests.thirdParty + li: a(href=`http://${host}`)= host + + p Requests have been made to #{ hosts.requests.thirdParty.length } distinct third-party host(s). + + h4 First-party potential web beacon hosts + + ol + each host in hosts.beacons.firstParty + li: a(href=`http://${host}`)= host + + if hosts.beacons.firstParty.length > 0 + p Potential first-party web beacons were sent to #{ hosts.beacons.firstParty.length } distinct host(s). Corresponding HTTP requests for first- and third-parties are listed in #[span.citation(data-cites="app:annex-beacons"): a(target="_parent", href="#app:annex-beacons") the Annex: All potential web beacons]. + else + p No first-party potential web beacons were found. + + h4 Third-party potential web beacon hosts + + ol + each host in hosts.beacons.thirdParty + li: a(href=`http://${host}`)= host + + if hosts.beacons.thirdParty.length > 0 + p Potential third-party web beacons were sent to #{ hosts.beacons.thirdParty.length } distinct host(s). Corresponding HTTP requests for first- and third-parties are listed in #[span.citation(data-cites="app:annex-beacons"): a(target="_parent", href="#app:annex-beacons") the Annex: All potential web beacons]. + else + p No third-party potential web beacons were found. + + h4(id="sec:unsecure-forms") Web forms with non-encrypted transmission + + if unsafeForms.length > 0 + table.unfase-webforms + colgroup + col + col + thead + tr + th # + th Web form ID + th Recipient URL + th HTTP method + tbody + each form, index in unsafeForms + tr + td= index + 1 + td= form.id + td= form.action + td= form.method + + p The evidence collection tool logged #{ unsafeForms.length } web forms that submit data potentially with no SSL encryption to a different web page. + else + p No web forms submitting data without SSL encryption were detected. + + h3(id="sec:persistent-data-analysis") Persistent data analysis + + p The evidence collection tool analysed cookies after the browsing session. Web pages can also use the persistent HTML5 #[em local storage]. #[span.citation(data-cites="sec:local-storage"): a(target="_parent", href="#sec:local-storage") The subsequent section] lists its content after the browsing. + + - var cookiesByStorage = groupBy(cookies, "firstPartyStorage"); + + each cookieList, index in {'first-party': cookiesByStorage['true'] || [], 'third-party': cookiesByStorage['false'] || []} + h4 Cookies linked to #{ index } hosts + + if cookieList.length > 0 + table.cookies(style="width: 100%") + colgroup + col(width="0%") + col(width="0%") + col(width="0%") + col(width="100%") + col(width="0%") + thead + tr + th.notrunc # + th.notrunc Host + th.notrunc Path + th.trunc Name + th.notrunc Expiry in days + tbody + each cookie, index in cookieList + tr + td.notrunc= index + 1 + td.notrunc: a( + href=`http://${cookie.domain}`, + title=`http://${cookie.domain}` + )= cookie.domain + td.notrunc: a( + href=`http://${cookie.domain}${cookie.path}`, + title=`http://${cookie.domain}${cookie.path}` + )= cookie.path + td.trunc(title=cookie.name)= cookie.name + td.notrunc + if cookie.session + em session + else + = cookie.expiresDays + + p In total, #{ cookieList.length } #{ index.toLowerCase() } cookie(s) were found. + else + p No #{ index.toLowerCase() } cookies were found. + + h4(id="sec:local-storage") Local storage + + if Object.keys(localStorage).length > 0 + table.local-storage(style="width: 100%") + colgroup + col(width="0%") + col(width="20%") + col(width="40%") + col(width="40%") + thead + tr + th.notrunc # + th.trunc Host + th.trunc Key + th.trunc Value + tbody + - let index = 1; + each storage, url in localStorage + each data, key in storage + tr + td.notrunc= index++ + td.trunc: a(href=url, title=url)= url.replace(/(^\w+:|^)\/\//, "") + td.trunc(title=key)= key + td.trunc.code: pre: code= JSON.stringify(data.value, null, 2) + else + p The local storage was found to be empty. + + if extra && extra[0] && extra[0].browser && extra[0].browser.extra_headers && extra[0].browser.extra_headers.dnt + h2(id="traffic-and-persistent-data-analysis-dnt") Traffic and persistent data analysis (DNT set) + + p The same web page(s) were browsed a second time consecutively between #{ new Date(extra[0].start_time || start_time).toLocaleString("en-GB") } and #{ new Date(extra[0].end_time || end_time).toLocaleString("en-GB") }. + + p During this second browsing session, the HTTP Header #[a(href="https://en.wikipedia.org/wiki/Do_Not_Track") Do Not Track] was explicitly set. + + h3(id="sec:traffic-analysis-dnt") Traffic analysis (DNT set) + + h4 First-party hosts (DNT set) + + ol + each host in extra[0].hosts.requests.firstParty + li: a(href=`http://${host}`)= host + + p Requests with DNT set have been made to #{ extra[0].hosts.requests.firstParty.length } distinct first-party host(s). + + h4 Third-party hosts (DNT set) + + ol + each host in extra[0].hosts.requests.thirdParty + li: a(href=`http://${host}`)= host + + p Requests with DNT set have been made to #{ extra[0].hosts.requests.thirdParty.length } distinct third-party host(s). + + h4 First-party potential web beacon hosts (DNT set) + + ol + each host in extra[0].hosts.beacons.firstParty + li: a(href=`http://${host}`)= host + + if extra[0].hosts.beacons.firstParty.length > 0 + p Potential first-party web beacons with DNT set were sent to #{ extra[0].hosts.beacons.firstParty.length } distinct host(s). + else + p No first-party potential web beacons with DNT set were found. + + h4 Third-party potential web beacon hosts (DNT set) + + ol + each host in extra[0].hosts.beacons.thirdParty + li: a(href=`http://${host}`)= host + + if extra[0].hosts.beacons.thirdParty.length > 0 + p Potential third-party web beacons with DNT set were sent to #{ extra[0].hosts.beacons.thirdParty.length } distinct host(s). + else + p No third-party potential web beacons with DNT set were found. + + h4(id="sec:unsecure-forms-dnt") Web forms with non-encrypted transmission (DNT set) + + if extra[0].unsafeForms && extra[0].unsafeForms.length > 0 + table.unfase-webforms + colgroup + col + col + thead + tr + th # + th Web form ID + th Recipient URL + th HTTP method + tbody + each form, index in extra[0].unsafeForms + tr + td= index + 1 + td= form.id + td= form.action + td= form.method + + p The evidence collection tool logged #{ extra[0].unsafeForms.length } web forms that submit data potentially with no SSL encryption to a different web page when DNT is set. + else + p No web forms submitting data without SSL encryption were detected when DNT is set. + + h3(id="sec:persistent-data-analysis-dnt") Persistent data analysis (DNT set) + + p The evidence collection tool analysed cookies and local storage after the DNT-enabled browsing session. + + - var cookiesByStorageDNT = groupBy(extra[0].cookies, "firstPartyStorage"); + + each cookieList, index in {'first-party': cookiesByStorageDNT['true'] || [], 'third-party': cookiesByStorageDNT['false'] || []} + h4 Cookies linked to #{ index } hosts (DNT set) + + if cookieList.length > 0 + table.cookies(style="width: 100%") + colgroup + col(width="0%") + col(width="0%") + col(width="0%") + col(width="100%") + col(width="0%") + thead + tr + th.notrunc # + th.notrunc Host + th.notrunc Path + th.trunc Name + th.notrunc Expiry in days + tbody + each cookie, index in cookieList + tr + td.notrunc= index + 1 + td.notrunc: a( + href=`http://${cookie.domain}`, + title=`http://${cookie.domain}` + )= cookie.domain + td.notrunc: a( + href=`http://${cookie.domain}${cookie.path}`, + title=`http://${cookie.domain}${cookie.path}` + )= cookie.path + td.trunc(title=cookie.name)= cookie.name + td.notrunc + if cookie.session + em session + else + = cookie.expiresDays + + p In total, #{ cookieList.length } #{ index.toLowerCase() } cookie(s) were found when DNT was set. + else + p No #{ index.toLowerCase() } cookies were found when DNT was set. + + h4(id="sec:local-storage-dnt") Local storage (DNT set) + + if extra[0].localStorage && Object.keys(extra[0].localStorage).length > 0 + table.local-storage(style="width: 100%") + colgroup + col(width="0%") + col(width="20%") + col(width="40%") + col(width="40%") + thead + tr + th.notrunc # + th.trunc Host + th.trunc Key + th.trunc Value + tbody + - let index = 1; + each storage, url in extra[0].localStorage + each data, key in storage + tr + td.notrunc= index++ + td.trunc: a(href=url, title=url)= url.replace(/(^\w+:|^)\/\//, "") + td.trunc(title=key)= key + td.trunc.code: pre: code= JSON.stringify(data.value, null, 2) + else + p The local storage was found to be empty when DNT was set. + + h1(id="app:annex") Annex + + h2(id="app:history") Browsing history + + p For the collection of evidence, the browser navigated consecutively to the following #{ browsing_history.length } web page(s): + + ol + each link in browsing_history + li: a(href=link)= link + + h2(id="app:annex-beacons") All potential web beacons + + p The data transmitted by beacons using HTTP GET parameters are decoded for improved readability and displayed beneath the beacon URL. + + each beaconsByList, listName in groupBy(beacons, 'listName') + h5(id=`annex-beacons-${listName}`)= listName + + table.adblock-findings(style="width: 100%") + colgroup + col(width="0%") + col(width="100%") + col(width="0%") + thead + tr + th.notrunc # + th.trunc Sample URL + th.notrunc Freq. + tbody + each beacon, index in beaconsByList + tr + td.notrunc= index + 1 + td.trunc(title=beacon.url)= beacon.url + td.notrunc= beacon.occurrances + if beacon.query + tr + td.notrunc + td.trunc.code(colspan=2): pre: code= JSON.stringify(beacon.query, null, 2).split("\n").slice(1, -1).join("\n").replace(/^ /gm, "") + + if testSSL + - var results = testSSL.scanResult[0]; + + h2(id="app:testssl") TestSSL scan + + p The following data stems from a #[a(href="https://testssl.sh/") TestSSL] scan. The severity ratings are automatically computed by the TestSSL software without considering the security requirements of the individual website. They do not reflect the opinions or views of the website evidence collector's authors. + + p.screen-only #[a(href="testssl/testssl.html") Click here] to check whether the full TestSSL scan report is available. + + table(width="100%") + colgroup + col + col + tbody + tr + td TestSSL version + td= testSSL.version + tr + td OpenSSL version + td= testSSL.openssl + tr + td Target host + td #{ results.targetHost } (#{ results.ip }) + + h3.unnumbered Protocols + + table(width="100%") + colgroup + col(style="width: 0%") + col(style="width: 100%") + col(style="width: 0%") + thead + tr + th Protocol + th Finding + th Severity + tbody + each protocol in results.protocols.sort(sortSeverity) + tr + td.notrunc= protocol.id + td= protocol.finding + td.notrunc= protocol.severity + + h3.unnumbered HTTPS/SSL vulnerabilities + + table(width="100%") + colgroup + col(style="width: 0%") + col(style="width: 80%") + col(style="width: 20%") + col(style="width: 0%") + thead + tr + th Vulnerability + th Finding + th CVE + th Severity + tbody + each vulnerability in results.vulnerabilities.sort(sortSeverity) + tr + td.notrunc= vulnerability.id + td.trunc(title=vulnerability.finding)= vulnerability.finding + td.trunc.trunc(title=vulnerability.cve) + if vulnerability.cve + each cve in vulnerability.cve.split(' ') + a( + href=`https://cve.mitre.org/cgi-bin/cvename.cgi?name=${cve}` + )= cve + | + td.notrunc= vulnerability.severity + + h3.unnumbered Cipher categories + + table(style="width: 100%") + colgroup + col(style="width: 0%") + col(style="width: 100%") + col(style="width: 0%") + col(style="width: 0%") + thead + tr + th Name + th Finding + th CWE + th Severity + tbody + each cipher in results.ciphers.sort(sortSeverity) + tr + td.notrunc= cipher.id + td.notrunc= cipher.finding + td.notrunc + if cipher.cwe + a( + href=`https://cwe.mitre.org/cgi-bin/jumpmenu.cgi?id=${cipher.cwe.replace('CWE-','')}` + )= cipher.cwe + td.notrunc= cipher.severity + + h3.unnumbered HTTP header responses + + table(style="width: 100%") + colgroup + col(style="width: 0%") + col(style="width: 100%") + col(style="width: 0%") + thead + tr + th Name + th Finding + th Severity + tbody + each response in results.headerResponse.sort(sortSeverity) + tr + td.notrunc= response.id + td.trunc(title=response.finding)= response.finding + td.notrunc= response.severity + + h2(id="app:glossary") Glossary + dl + dt Do Not Track (DNT for short, HTTP) + dd The Do Not Track header is the proposed HTTP header field DNT, which requests that a web service does not track its individual visitors. Note that this request cannot be enforced by technical means on the visitors’ side. It is upon the web service to take the DNT header field into account. + dt Filter Lists + dd Browser extensions commonly referred to as #[em Adblockers] have been developed to block the loading of advertisements based on filter lists. Over time, these filter lists have been extended to also block the loading of web page elements associated with tracking web page visitors. For this evidence collection, publicly available tracking filter lists are used to identify web page elements that may track the web page visitors. + dt First-Party + dd In this document, #[em first-party] is a classification for resource links, web beacons, and cookies. To be considered first party, the resource’s domain must match the domain of the inspected web service or other configured first-party domains. Note that the resource path must also be within the path of the web service to be classified as first-party. + dt Host (HTTP) + dd The HTTP #[em host] is the computer that receives and responds to browser requests for web pages. + dt Local Storage (HTML5) + dd Most web browsers allow web pages to store data locally in the browser profile. This #[em local storage] is specific to the website and persists through browser shutdowns. As embedded third-party resources may also have access to first-party local storage, it is classified both as first- and third-party. + dt Redirect (HTTP) + dd A request for a web page may be answered with a new location (URL) to be requested instead. These HTTP #[em redirects] can be used to enforce the use of HTTPS. When visitors request an HTTP web page, they are redirected to the corresponding HTTPS web page. + dt Request (HTTP) + dd To download and display a web page identified by a URL, browsers send HTTP #[em requests] with the URL to the host computer specified as part of the URL. + dt Third-Party + dd Links, web beacons and cookies that are not #[em first-party] (see above) are classified as #[em third-party]. + dt Web Beacon + dd A web beacon is one of various techniques used on web pages to unobtrusively (usually invisibly) track web page visitors. A web beacon can be implemented as a 1x1 pixel image, a transparent image, or an empty file requested alongside other resources when a web page is loaded. + dt Web Beacon Host + dd The #[em host] in the URL of a #[em request] of a #[em web beacon] is referred to as the #[em web beacon host]. + + script. + // @license magnet:?xt=urn:btih:d3d9a9a6595521f9666a5e94cc830dab83b65699&dn=expat.txt Expat + // + // AnchorJS - v5.0.0 - 2023-01-18 + // https://www.bryanbraun.com/anchorjs/ + // Copyright (c) 2023 Bryan Braun; Licensed MIT + // + // @license magnet:?xt=urn:btih:d3d9a9a6595521f9666a5e94cc830dab83b65699&dn=expat.txt Expat + !(function (A, e) { + "use strict"; + "function" == typeof define && define.amd ? define([], e) : "object" == typeof module && module.exports ? (module.exports = e()) : ((A.AnchorJS = e()), (A.anchors = new A.AnchorJS())); + })(globalThis, function () { + "use strict"; + return function (A) { + function u(A) { + (A.icon = Object.prototype.hasOwnProperty.call(A, "icon") ? A.icon : ""), (A.visible = Object.prototype.hasOwnProperty.call(A, "visible") ? A.visible : "hover"), (A.placement = Object.prototype.hasOwnProperty.call(A, "placement") ? A.placement : "right"), (A.ariaLabel = Object.prototype.hasOwnProperty.call(A, "ariaLabel") ? A.ariaLabel : "Anchor"), (A.class = Object.prototype.hasOwnProperty.call(A, "class") ? A.class : ""), (A.base = Object.prototype.hasOwnProperty.call(A, "base") ? A.base : ""), (A.truncate = Object.prototype.hasOwnProperty.call(A, "truncate") ? Math.floor(A.truncate) : 64), (A.titleText = Object.prototype.hasOwnProperty.call(A, "titleText") ? A.titleText : ""); + } + function d(A) { + var e; + if ("string" == typeof A || A instanceof String) e = [].slice.call(document.querySelectorAll(A)); + else { + if (!(Array.isArray(A) || A instanceof NodeList)) throw new TypeError("The selector provided to AnchorJS was invalid."); + e = [].slice.call(A); + } + return e; + } + (this.options = A || {}), + (this.elements = []), + u(this.options), + (this.add = function (A) { + var e, + t, + o, + i, + n, + s, + a, + r, + l, + c, + h, + p = []; + if ((u(this.options), 0 !== (e = d((A = A || "h2, h3, h4, h5, h6"))).length)) { + for ( + null === document.head.querySelector("style.anchorjs") && (((A = document.createElement("style")).className = "anchorjs"), A.appendChild(document.createTextNode("")), void 0 === (h = document.head.querySelector('[rel="stylesheet"],style')) ? document.head.appendChild(A) : document.head.insertBefore(A, h), A.sheet.insertRule(".anchorjs-link{opacity:0;text-decoration:none;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}", A.sheet.cssRules.length), A.sheet.insertRule(":hover>.anchorjs-link,.anchorjs-link:focus{opacity:1}", A.sheet.cssRules.length), A.sheet.insertRule("[data-anchorjs-icon]::after{content:attr(data-anchorjs-icon)}", A.sheet.cssRules.length), A.sheet.insertRule('@font-face{font-family:anchorjs-icons;src:url(data:n/a;base64,AAEAAAALAIAAAwAwT1MvMg8yG2cAAAE4AAAAYGNtYXDp3gC3AAABpAAAAExnYXNwAAAAEAAAA9wAAAAIZ2x5ZlQCcfwAAAH4AAABCGhlYWQHFvHyAAAAvAAAADZoaGVhBnACFwAAAPQAAAAkaG10eASAADEAAAGYAAAADGxvY2EACACEAAAB8AAAAAhtYXhwAAYAVwAAARgAAAAgbmFtZQGOH9cAAAMAAAAAunBvc3QAAwAAAAADvAAAACAAAQAAAAEAAHzE2p9fDzz1AAkEAAAAAADRecUWAAAAANQA6R8AAAAAAoACwAAAAAgAAgAAAAAAAAABAAADwP/AAAACgAAA/9MCrQABAAAAAAAAAAAAAAAAAAAAAwABAAAAAwBVAAIAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAMCQAGQAAUAAAKZAswAAACPApkCzAAAAesAMwEJAAAAAAAAAAAAAAAAAAAAARAAAAAAAAAAAAAAAAAAAAAAQAAg//0DwP/AAEADwABAAAAAAQAAAAAAAAAAAAAAIAAAAAAAAAIAAAACgAAxAAAAAwAAAAMAAAAcAAEAAwAAABwAAwABAAAAHAAEADAAAAAIAAgAAgAAACDpy//9//8AAAAg6cv//f///+EWNwADAAEAAAAAAAAAAAAAAAAACACEAAEAAAAAAAAAAAAAAAAxAAACAAQARAKAAsAAKwBUAAABIiYnJjQ3NzY2MzIWFxYUBwcGIicmNDc3NjQnJiYjIgYHBwYUFxYUBwYGIwciJicmNDc3NjIXFhQHBwYUFxYWMzI2Nzc2NCcmNDc2MhcWFAcHBgYjARQGDAUtLXoWOR8fORYtLTgKGwoKCjgaGg0gEhIgDXoaGgkJBQwHdR85Fi0tOAobCgoKOBoaDSASEiANehoaCQkKGwotLXoWOR8BMwUFLYEuehYXFxYugC44CQkKGwo4GkoaDQ0NDXoaShoKGwoFBe8XFi6ALjgJCQobCjgaShoNDQ0NehpKGgobCgoKLYEuehYXAAAADACWAAEAAAAAAAEACAAAAAEAAAAAAAIAAwAIAAEAAAAAAAMACAAAAAEAAAAAAAQACAAAAAEAAAAAAAUAAQALAAEAAAAAAAYACAAAAAMAAQQJAAEAEAAMAAMAAQQJAAIABgAcAAMAAQQJAAMAEAAMAAMAAQQJAAQAEAAMAAMAAQQJAAUAAgAiAAMAAQQJAAYAEAAMYW5jaG9yanM0MDBAAGEAbgBjAGgAbwByAGoAcwA0ADAAMABAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAH//wAP) format("truetype")}', A.sheet.cssRules.length)), + h = document.querySelectorAll("[id]"), + t = [].map.call(h, function (A) { + return A.id; + }), + i = 0; + i < e.length; + i++ + ) + if (this.hasAnchorJSLink(e[i])) p.push(i); + else { + if (e[i].hasAttribute("id")) o = e[i].getAttribute("id"); + else if (e[i].hasAttribute("data-anchor-id")) o = e[i].getAttribute("data-anchor-id"); + else { + for (r = a = this.urlify(e[i].textContent), s = 0; (n = t.indexOf((r = void 0 !== n ? a + "-" + s : r))), (s += 1), -1 !== n; ); + (n = void 0), t.push(r), e[i].setAttribute("id", r), (o = r); + } + ((l = document.createElement("a")).className = "anchorjs-link " + this.options.class), l.setAttribute("aria-label", this.options.ariaLabel), l.setAttribute("data-anchorjs-icon", this.options.icon), this.options.titleText && (l.title = this.options.titleText), (c = document.querySelector("base") ? window.location.pathname + window.location.search : ""), (c = this.options.base || c), (l.href = c + "#" + o), "always" === this.options.visible && (l.style.opacity = "1"), "" === this.options.icon && ((l.style.font = "1em/1 anchorjs-icons"), "left" === this.options.placement) && (l.style.lineHeight = "inherit"), "left" === this.options.placement ? ((l.style.position = "absolute"), (l.style.marginLeft = "-1.25em"), (l.style.paddingRight = ".25em"), (l.style.paddingLeft = ".25em"), e[i].insertBefore(l, e[i].firstChild)) : ((l.style.marginLeft = ".1875em"), (l.style.paddingRight = ".1875em"), (l.style.paddingLeft = ".1875em"), e[i].appendChild(l)); + } + for (i = 0; i < p.length; i++) e.splice(p[i] - i, 1); + this.elements = this.elements.concat(e); + } + return this; + }), + (this.remove = function (A) { + for (var e, t, o = d(A), i = 0; i < o.length; i++) (t = o[i].querySelector(".anchorjs-link")) && (-1 !== (e = this.elements.indexOf(o[i])) && this.elements.splice(e, 1), o[i].removeChild(t)); + return this; + }), + (this.removeAll = function () { + this.remove(this.elements); + }), + (this.urlify = function (A) { + var e = document.createElement("textarea"); + return ( + (e.innerHTML = A), + (A = e.value), + this.options.truncate || u(this.options), + A.trim() + .replace(/'/gi, "") + .replace(/[& +$,:;=?@"#\u007B\u007D|^~[`%!'<>\]./()*\\\n\t\b\v\u00A0]/g, "-") + .replace(/-{2,}/g, "-") + .substring(0, this.options.truncate) + .replace(/^-+|-+$/gm, "") + .toLowerCase() + ); + }), + (this.hasAnchorJSLink = function (A) { + var e = A.firstChild && -1 < (" " + A.firstChild.className + " ").indexOf(" anchorjs-link "), + A = A.lastChild && -1 < (" " + A.lastChild.className + " ").indexOf(" anchorjs-link "); + return e || A || !1; + }); + }; + }); + // @license-end + + // Enable links for selected headers + var anchors = new AnchorJS(); + anchors.options = { + placement: "right", + /* visible: 'always', */ + /* icon: '¶', */ + }; + anchors.add(":not(header) > h1, :not(header) > h2, h3, h4, h5, h6"); + + script. + function highlighter(item) { + item.addEventListener( + "dblclick", + function () { + item.classList.toggle("highlighted"); + }, + false, + ); + } + // highlighting for table cells + [].forEach.call(document.getElementsByTagName("td"), highlighter); + // highlighting for list items + [].forEach.call(document.getElementsByTagName("li"), highlighter); diff --git a/src/reporter/reporter.ts b/src/reporter/reporter.ts index c0e081ce749a9c3260eceea08ceeb769585f2b1d..f798b8a1eb1c660d200ed314b5525f6dae667921 100644 --- a/src/reporter/reporter.ts +++ b/src/reporter/reporter.ts @@ -44,7 +44,7 @@ export interface ReporterOptions { usePandoc: boolean; htmlTemplate?: string; officeTemplate?: string; - extraFiles: string[]; + extraFiles: any[]; } export class Reporter { diff --git a/src/server/runCollection.ts b/src/server/runCollection.ts index 406b359dfd8d950ae239d4246d1c5719bcc79584..9d05195feb535b48f7b1e873dbbe6ad6913e86dd 100644 --- a/src/server/runCollection.ts +++ b/src/server/runCollection.ts @@ -3,6 +3,11 @@ import { Collector } from "../collector/index.js"; import Inspector from "../inspector/inspector.js"; import { Cookie } from "./server.js"; import { Logger } from "winston"; +import path from "path"; +import { fileURLToPath } from "url"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); export interface RunCollectionArguments { website_url: string; @@ -38,9 +43,15 @@ export async function runCollection( return inspector.run(); } +/* + * Generates the report for the specified template and output as HTML and PDF. + * The PDF is encoded as Base64. + */ export async function generateHtmlAndPdf( - inspectionOutput: object, logger: Logger, + inspectionOutput: object, + templatePath: string, + additionalInspectionOutput?: [object], ) { let reporterArgs: ReporterOptions = { printHtmlToConsole: false, @@ -49,11 +60,16 @@ export async function generateHtmlAndPdf( outputPath: undefined, usePandoc: false, printYamlToConsole: false, - extraFiles: [], + extraFiles: additionalInspectionOutput ?? [], }; const reporter = new Reporter(reporterArgs, logger); - let html = reporter.generateHtmlReport(inspectionOutput, "inspection.html"); + let html = reporter.generateHtmlReport( + inspectionOutput, + "inspection.html", + templatePath, + ); + let pdfBuffer = await reporter.convertHtmlToPdfInMemory(html); return { html: html, @@ -61,6 +77,25 @@ export async function generateHtmlAndPdf( }; } +/* + * The function returns the resolved paths for the available templates. + * This is necessary to enable other libraries to use the templates contained in the WEC as there paths have to be resolved. + */ +export function availableTemplates(): { + dntTemplate: string; + regularTemplate: string; +} { + const dntComparisonTemplate = path.join( + __dirname, + "../assets/template-dnt-comparison.pug", + ); + const regularTemplate = path.join(__dirname, "../assets/template.pug"); + return { + dntTemplate: dntComparisonTemplate, + regularTemplate: regularTemplate, + }; +} + /** * Constructs a JSON object containing all Arguments as it is expected by the underlying implementation. */ diff --git a/src/server/server.ts b/src/server/server.ts index 6464404df31d72cedeb40bcf7760ffe69c31bc18..4766eb80279348487e2157c11dd19e4146b7c0e9 100644 --- a/src/server/server.ts +++ b/src/server/server.ts @@ -7,6 +7,7 @@ import express, { } from "express"; import bodyParser from "body-parser"; import { + availableTemplates, generateHtmlAndPdf, runCollection, RunCollectionArguments, @@ -122,9 +123,11 @@ function configureRoutes(browser_options: any[]): Router { ); let htmlAndPdf = await generateHtmlAndPdf( - collectionOutput, requestLogger, + collectionOutput, + availableTemplates().regularTemplate, ); + res.send(htmlAndPdf); requestLogger.info("Finished serving request"); } catch (e: any) {