Fix all download failures claiming that the download was disabled by x-block-origin. (#1513)

Author: JavierMatosD

azure-pipelines/end-to-end-tests-dir/asset-caching.ps1

@@ -57,6 +57,8 @@ $actual = $actual -replace "`r`n", "`n"
 $expected = @(
 "A suitable version of .* was not found \(required v[0-9\.]+\)."
 "Asset cache miss; downloading from .*"
+"Downloading .*"
+"Successfully downloaded .*."
 "Successfully stored .* to .*."
 ) -join "`n"
 
@@ -82,26 +84,116 @@ if (-not ($actual -match $expected)) {
 # Testing asset caching && x-block-orgin promises when --debug is passed (enabled)
 Refresh-TestRoot
 $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin", "--downloads-root=$DownloadsRoot", "--debug"))
-$actual = $actual -replace "`r`n", "`n"
-
-# Define the regex pattern that accounts for multiline input
-$expectedPattern = "(?s)" +
-                   ".*\[DEBUG\] External asset downloads are blocked \(x-block-origin is enabled\)\.\.\.?" +
-                   ".*\[DEBUG\] Asset caching is enabled\..*"
-
-if (-not ($actual -match $expectedPattern)) {
+if (-not ($actual.Contains("[DEBUG] External asset downloads are blocked (x-block-origin is enabled)") -and $actual.Contains("[DEBUG] Asset caching is enabled."))) {
     throw "Failure: couldn't find expected debug promises (asset caching enabled + x-block-origin enabled)"
 }
 
 # Testing asset caching && x-block-orgin promises when --debug is passed (disabled)
 Refresh-TestRoot
 $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear", "--downloads-root=$DownloadsRoot", "--debug"))
+if (-not ($actual.Contains("[DEBUG] External asset downloads are allowed (x-block-origin is disabled)") -and $actual.Contains("[DEBUG] Asset cache is not configured"))) {
+    throw "Failure: couldn't find expected debug promises (asset caching disabled + x-block-origin disabled)"
+}
+
+# azurl (no), x-block-origin (no), asset-cache (n/a), download (fail)
+# Expected: Download failure message, nothing about asset caching
+Refresh-TestRoot
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://localhost:1234/foobar.html"))
+if (-not ($actual.Contains("error: https://localhost:1234/foobar.html: curl failed to download with exit code 7"))) {
+    throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (fail)"
+}
+
+#azurl (no), x-block-origin (no), asset-cache (n/a), download (sha-mismatch)
+#Expected: Download message with the "you might need to configure a proxy" message and with expected/actual sha
+Refresh-TestRoot
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b", "--url", "https://example.com"))
+if (-not ($actual.Contains("Failed to download example3.html.") -and
+          $actual.Contains("If you are using a proxy, please ensure your proxy settings are correct.") -and
+          $actual.Contains("error: File does not have the expected hash:") -and
+          $actual.Contains("url: https://example.com") -and
+          $actual.Contains("Expected hash: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b") -and
+          $actual.Contains("Actual hash: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a"))) {
+    throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (sha-mismatch)"
+}
+
+# azurl (no), x-block-origin (no), asset-cache (n/a), download (succeed)
+# Expected: Download success message, nothing about asset caching
+Refresh-TestRoot
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com"))
+if (-not ($actual.Contains("Downloading example3.html") -and
+          $actual.Contains("Successfully downloaded example3.html."))) {
+    throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (succeed)"
+}
+
+# azurl (no), x-block-origin (yes), asset-cache (n/a), download (n/a)
+# Expected: Download failure message, nothing about asset caching, x-block-origin complaint
+Refresh-TestRoot
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=clear;x-block-origin"))
+if (-not ($actual.Contains("error: Missing example3.html and downloads are blocked by x-block-origin."))) {
+    throw "Failure: azurl (no), x-block-origin (yes), asset-cache (n/a), download (n/a)"
+}
+
+# azurl (yes), x-block-origin (no), asset-cache (miss), download (fail)
+# Expected: Download failure message, asset cache named, nothing about x-block-origin
+Refresh-TestRoot
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://localhost:1234/foobar.html", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite"))
+if (-not ($actual.Contains("Asset cache miss; downloading from https://localhost:1234/foobar.html") -and
+          $actual.Contains("Downloading example3.html") -and
+          $actual.Contains("error: file://$AssetCache") -and
+          $actual.Contains("curl failed to download with exit code 37"))) {
+    throw "Failure: azurl (yes), x-block-origin (no), asset-cache (miss), download (fail)"
+}
+
+# azurl (yes), x-block-origin (no), asset-cache (hit), download (n/a)
+# Expected: Download success message, asset cache named, nothing about x-block-origin
+Refresh-TestRoot
 $actual = $actual -replace "`r`n", "`n"
+Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite"))
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite"))
+if (-not ($actual.Contains("Asset cache hit for example3.html; downloaded from: file://$AssetCache"))) {
+    throw "Failure: azurl (yes), x-block-origin (no), asset-cache (hit), download (n/a)"
+}
+
+# azurl (yes), x-block-origin (no), asset-cache (miss), download (sha-mismatch)
+# Expected: Download message with "you might need to configure a proxy" and expected/actual sha
+Refresh-TestRoot
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite"))
+if (-not ($actual.Contains("Asset cache miss; downloading from https://example.com") -and
+          $actual.Contains("Downloading example3.html") -and
+          $actual.Contains("error: file://$AssetCache") -and
+          $actual.Contains("curl failed to download with exit code 37") -and
+          $actual.Contains("error: File does not have the expected hash:") -and
+          $actual.Contains("url: https://example.com") -and
+          $actual.Contains("Expected hash: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b") -and
+          $actual.Contains("Actual hash: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a"))) {
+    throw "Failure: azurl (yes), x-block-origin (no), asset-cache (miss), download (sha-mismatch)"
+}
 
-$expectedPattern = "(?s)" +
-                   ".*\[DEBUG\] External asset downloads are allowed \(x-block-origin is disabled\)\.\.\.?" +
-                   ".*\[DEBUG\] Asset cache is not configured.*"
+# azurl (yes), x-block-origin (no), asset-cache (miss), download (succeed)
+# Expected: Download success message, asset cache upload, nothing about x-block-origin
+Refresh-TestRoot
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite"))
+if (-not ($actual.Contains("Asset cache miss; downloading from https://example.com") -and
+          $actual.Contains("Downloading example3.html") -and
+          $actual.Contains("Successfully downloaded example3.html.") -and
+          $actual.Contains("Successfully stored example3.html to file://$AssetCache"))) {
+    throw "Failure: azurl (yes), x-block-origin (no), asset-cache (miss), download (succeed)"
+}
 
-if (-not ($actual -match $expectedPattern)) {
-    throw "Failure: couldn't find expected debug promises (asset caching disabled + x-block-origin disabled)"
+# azurl (yes), x-block-origin (yes), asset-cache (miss), download (n/a)
+# Expected: Download failure message, which asset cache was tried, x-block-origin complaint
+Refresh-TestRoot
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin"))
+if (-not ($actual.Contains("Asset cache miss for example3.html and downloads are blocked by x-block-origin.") -and
+          $actual.Contains("error: Missing example3.html and downloads are blocked by x-block-origin."))) {
+    throw "Failure: azurl (yes), x-block-origin (yes), asset-cache (miss), download (n/a)"
+}
+
+# azurl (yes), x-block-origin (yes), asset-cache (hit), download (n/a)
+# Expected: Download success message, asset cache named, nothing about x-block-origin
+Refresh-TestRoot
+Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite"))
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin"))
+if (-not ($actual.Contains("Asset cache hit for example3.html; downloaded from: file://$AssetCache"))) {
+    throw "Failure: azurl (yes), x-block-origin (yes), asset-cache (hit), download (n/a)"
 }

include/vcpkg/base/message-data.inc.h

@@ -248,6 +248,11 @@ DECLARE_MESSAGE(ArtifactsSwitchX86, (), "", "Forces host detection to x86 when a
 DECLARE_MESSAGE(ArtifactsSwitchWindows, (), "", "Forces host detection to Windows when acquiring artifacts")
 DECLARE_MESSAGE(AssetCacheHit, (msg::path, msg::url), "", "Asset cache hit for {path}; downloaded from: {url}")
 DECLARE_MESSAGE(AssetCacheMiss, (msg::url), "", "Asset cache miss; downloading from {url}")
+DECLARE_MESSAGE(AssetCacheMissBlockOrigin,
+                (msg::path),
+                "x-block-origin is a vcpkg term. Do not translate",
+                "Asset cache miss for {path} and downloads are blocked by x-block-origin.")
+DECLARE_MESSAGE(DownloadSuccesful, (msg::path), "", "Successfully downloaded {path}.")
 DECLARE_MESSAGE(DownloadingUrl, (msg::url), "", "Downloading {url}")
 DECLARE_MESSAGE(AssetCacheProviderAcceptsNoArguments,
                 (msg::value),
@@ -1046,6 +1051,22 @@ DECLARE_MESSAGE(DownloadFailedStatusCode,
                 (msg::url, msg::value),
                 "{value} is an HTTP status code",
                 "{url}: failed: status code {value}")
+DECLARE_MESSAGE(DownloadFailedProxySettings,
+                (msg::path, msg::url),
+                "",
+                "Failed to download {path}.\nIf you are using a proxy, please ensure your proxy settings are "
+                "correct.\nPossible causes are:\n"
+                "1. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable "
+                "to `https//address:port`.\nThis is not correct, because `https://` prefix "
+                "claims the proxy is an HTTPS proxy, while your proxy (v2ray, shadowsocksr, etc...) is an HTTP proxy.\n"
+                "Try setting `http://address:port` to both HTTP_PROXY and HTTPS_PROXY instead.\n"
+                "2. If you are using Windows, vcpkg will automatically use your Windows IE Proxy Settings "
+                "set by your proxy software. See, {url}\n"
+                "The value set by your proxy might be wrong, or have same `https://` prefix issue.\n"
+                "3. Your proxy's remote server is our of service.\n"
+                "If you've tried directly download the link, and believe this is not a temporay download server "
+                "failure, please submit an issue at https://github.com/Microsoft/vcpkg/issues\n"
+                "to report this upstream download server failure.")
 DECLARE_MESSAGE(DownloadingPortableToolVersionX,
                 (msg::tool_name, msg::version),
                 "",
@@ -1240,10 +1261,6 @@ DECLARE_MESSAGE(MissingShaVariable,
                 (),
                 "{{sha}} should not be translated",
                 "The {{sha}} variable must be used in the template if other variables are used.")
-DECLARE_MESSAGE(AssetCacheMissBlockOrigin,
-                (msg::path),
-                "x-block-origin is a vcpkg term. Do not translate",
-                "Asset cache miss for {path} and downloads are blocked by x-block-origin.")
 DECLARE_MESSAGE(FailedToExtract, (msg::path), "", "Failed to extract \"{path}\":")
 DECLARE_MESSAGE(FailedToFetchRepo, (msg::url), "", "Failed to fetch {url}.")
 DECLARE_MESSAGE(FailedToFindPortFeature,

locales/messages.json

@@ -595,12 +595,16 @@
   "_DownloadFailedCurl.comment": "An example of {url} is https://github.com/microsoft/vcpkg. An example of {exit_code} is 127.",
   "DownloadFailedHashMismatch": "File does not have the expected hash:\nurl: {url}\nFile: {path}\nExpected hash: {expected}\nActual hash: {actual}",
   "_DownloadFailedHashMismatch.comment": "{expected} and {actual} are SHA512 hashes in hex format. An example of {url} is https://github.com/microsoft/vcpkg. An example of {path} is /foo/bar.",
+  "DownloadFailedProxySettings": "Failed to download {path}.\nIf you are using a proxy, please ensure your proxy settings are correct.\nPossible causes are:\n1. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to `https//address:port`.\nThis is not correct, because `https://` prefix claims the proxy is an HTTPS proxy, while your proxy (v2ray, shadowsocksr, etc...) is an HTTP proxy.\nTry setting `http://address:port` to both HTTP_PROXY and HTTPS_PROXY instead.\n2. If you are using Windows, vcpkg will automatically use your Windows IE Proxy Settings set by your proxy software. See, {url}\nThe value set by your proxy might be wrong, or have same `https://` prefix issue.\n3. Your proxy's remote server is our of service.\nIf you've tried directly download the link, and believe this is not a temporay download server failure, please submit an issue at https://github.com/Microsoft/vcpkg/issues\nto report this upstream download server failure.",
+  "_DownloadFailedProxySettings.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.",
   "DownloadFailedRetrying": "Download failed -- retrying after {value}ms",
   "_DownloadFailedRetrying.comment": "{value} is a number of milliseconds",
   "DownloadFailedStatusCode": "{url}: failed: status code {value}",
   "_DownloadFailedStatusCode.comment": "{value} is an HTTP status code An example of {url} is https://github.com/microsoft/vcpkg.",
   "DownloadRootsDir": "Downloads directory (default: {env_var})",
   "_DownloadRootsDir.comment": "An example of {env_var} is VCPKG_DEFAULT_TRIPLET.",
+  "DownloadSuccesful": "Successfully downloaded {path}.",
+  "_DownloadSuccesful.comment": "An example of {path} is /foo/bar.",
   "DownloadWinHttpError": "{url}: {system_api} failed with exit code {exit_code}",
   "_DownloadWinHttpError.comment": "An example of {system_api} is CreateProcessW. An example of {exit_code} is 127. An example of {url} is https://github.com/microsoft/vcpkg.",
   "DownloadedSources": "Downloaded sources for {spec}",