diff --git a/.bazelignore b/.bazelignore index f159e245c..f2bd3560c 100644 --- a/.bazelignore +++ b/.bazelignore @@ -13,6 +13,7 @@ examples/semanticdb examples/testing/multi_frameworks_toolchain examples/testing/scalatest_repositories examples/testing/specs2_junit_repositories +test/compiler_sources_integrity test/proto_cross_repo_boundary/repo test_cross_build third_party/test/example_external_workspace diff --git a/.gitignore b/.gitignore index 74a9a64ad..6fa1d4d5c 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,9 @@ test/semanticdb/tempsrc # From scripts/create_repository.py repository-artifacts.json +# From scripts/*.py +**/__pycache__/ + # Until it settles down **/MODULE.bazel.lock @@ -25,3 +28,4 @@ tmp/ # Not required by tests. deps/latest/.bazelversion + diff --git a/protoc/private/protoc_integrity.bzl b/protoc/private/protoc_integrity.bzl index 2fc87f9c6..512609ed9 100644 --- a/protoc/private/protoc_integrity.bzl +++ b/protoc/private/protoc_integrity.bzl @@ -20,7 +20,6 @@ PROTOC_VERSIONS = [ "29.1", "29.0", ] - PROTOC_BUILDS = { "linux-aarch_64": { "exec_compat": [ diff --git a/scala/private/macros/compiler_sources_integrity.bzl b/scala/private/macros/compiler_sources_integrity.bzl new file mode 100644 index 000000000..9c690f2ef --- /dev/null +++ b/scala/private/macros/compiler_sources_integrity.bzl @@ -0,0 +1,276 @@ +"""Scala compiler source JAR integrity metadata. + +Generated and updated by scripts/update_compiler_sources_integrity.py. +""" + +URL_PREFIX = "https://repo1.maven.org/maven2/org/scala-lang/" +URL_SUFFIX_BY_MAJOR_VERSION = { + "2": "scala-compiler/{version}/scala-compiler-{version}-sources.jar", + "3": "scala3-compiler_3/{version}/scala3-compiler_3-{version}-sources.jar", +} +COMPILER_SOURCES = { + "2.11.12": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.11.12/scala-compiler-2.11.12-sources.jar", + "integrity": "sha256-1XeX/jmC1p1W1DIEZFn1ty6HpCIXDZjPKVw7G76T9FY=", + }, + "2.12.1": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.12.1/scala-compiler-2.12.1-sources.jar", + "integrity": "sha256-omHm4mF8PoxvsvZwNy3eLrdjHy8Rl3IQZQodb9P+yS8=", + }, + "2.12.2": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.12.2/scala-compiler-2.12.2-sources.jar", + "integrity": "sha256-MO5/7KveqWSZw8l+wtCcqQ5mlL7y044hO91z7ubL+PY=", + }, + "2.12.3": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.12.3/scala-compiler-2.12.3-sources.jar", + "integrity": "sha256-ELU1rglNSGnO8gEQb+akaWDiRdtvTHgSWOxo1u/nwto=", + }, + "2.12.4": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.12.4/scala-compiler-2.12.4-sources.jar", + "integrity": "sha256-Z10eXhY/TbH4vemyDtezDV5uY14YhVyw5PO15nKohRI=", + }, + "2.12.5": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.12.5/scala-compiler-2.12.5-sources.jar", + "integrity": "sha256-q/RhC/8yR4sWTTNlOqmiU8FQnQzZUFAQqyXaqfnXF/Y=", + }, + "2.12.6": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.12.6/scala-compiler-2.12.6-sources.jar", + "integrity": "sha256-0+nXzHtQyJZ2SBlZzrvyMSdYY8n3QQLeKCUNyS/9Sm8=", + }, + "2.12.7": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.12.7/scala-compiler-2.12.7-sources.jar", + "integrity": "sha256-Z2+YZe7hCLcD2BKiG6C9lTDhZhV9QxFxzcGWTTWdUU8=", + }, + "2.12.8": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.12.8/scala-compiler-2.12.8-sources.jar", + "integrity": "sha256-UXBnEaE5nwYgZ0WQfxiIFcMfLdlbObAfDiwvx4sOfjs=", + }, + "2.12.9": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.12.9/scala-compiler-2.12.9-sources.jar", + "integrity": "sha256-qsRwgFYVGmRUretuPdN5on8yZqYaC/qYbvJ4SmO4zDo=", + }, + "2.12.10": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.12.10/scala-compiler-2.12.10-sources.jar", + "integrity": "sha256-u2ldGwXFI5u5PvSwcCOZOlnfBzumFQ9UFyXYHxzCPk0=", + }, + "2.12.11": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.12.11/scala-compiler-2.12.11-sources.jar", + "integrity": "sha256-c7IwLsqbj8P8RzUvmxQGgzUMb3DgpKgdcDd5EvQlVmE=", + }, + "2.12.12": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.12.12/scala-compiler-2.12.12-sources.jar", + "integrity": "sha256-jsBFrSOoWJKdxVmcxCNkz1guHcNU0ZeHrJRZG0g5pw4=", + }, + "2.12.13": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.12.13/scala-compiler-2.12.13-sources.jar", + "integrity": "sha256-Iim2ccH0ge9SvLoZurhZMwNm+pFcq8p+BsAa3QLlzCE=", + }, + "2.12.14": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.12.14/scala-compiler-2.12.14-sources.jar", + "integrity": "sha256-Z77QNjCIh1NWvxspYOhjh31pdn68u1879AQ4Fc6LhlI=", + }, + "2.12.15": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.12.15/scala-compiler-2.12.15-sources.jar", + "integrity": "sha256-ZfeD8fvvfeZhIk9gesB8oDxdGaz9t/IjT/je8eebXNg=", + }, + "2.12.16": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.12.16/scala-compiler-2.12.16-sources.jar", + "integrity": "sha256-lcIXzIfuhGs5mQ4KnCc4JKOE3/usV9+E1Gb4Zt9Kkeo=", + }, + "2.12.17": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.12.17/scala-compiler-2.12.17-sources.jar", + "integrity": "sha256-YSVSUhiKe3xKmApNoabdxmrhePulRde4M3E0/1ew2xM=", + }, + "2.12.18": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.12.18/scala-compiler-2.12.18-sources.jar", + "integrity": "sha256-957oDxQCGCU/KjjJ1z+Km1UtBq/Oel9hzwgHmjiOId8=", + }, + "2.12.19": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.12.19/scala-compiler-2.12.19-sources.jar", + "integrity": "sha256-ccZCNcOUkOeGX3q5qgShbJmscgEF9totOoc7ikAILCM=", + }, + "2.12.20": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.12.20/scala-compiler-2.12.20-sources.jar", + "integrity": "sha256-B8FAyBh6e8vPznUJ608XBPy/oSrTx3mPjdyrlHfcUCg=", + }, + "2.13.0": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.13.0/scala-compiler-2.13.0-sources.jar", + "integrity": "sha256-CJi/+Ctl9YPhqHLK3R8H43t9oQyop2N2chZiutCobOY=", + }, + "2.13.1": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.13.1/scala-compiler-2.13.1-sources.jar", + "integrity": "sha256-RxPRkGxoLKSDoQ4rN70C2YCkZXlIeBlnJqg1aqhaDO0=", + }, + "2.13.2": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.13.2/scala-compiler-2.13.2-sources.jar", + "integrity": "sha256-pfLrYtjcCtdf8afYuhaYzyyBjOzo4nTcdo0Hmskbv8Q=", + }, + "2.13.3": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.13.3/scala-compiler-2.13.3-sources.jar", + "integrity": "sha256-26lichdpBUfaenDzTZIjPRtaRkLsY4MsCpmSD4NxHF4=", + }, + "2.13.4": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.13.4/scala-compiler-2.13.4-sources.jar", + "integrity": "sha256-pIEXIkOwO0Q84iuzUS7TIbazmKEK4RFsF6Gn6FJsOow=", + }, + "2.13.5": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.13.5/scala-compiler-2.13.5-sources.jar", + "integrity": "sha256-CuLhwjMtDnpDfJdhKLoCsbTOujByGyCnpllKjqrpbU8=", + }, + "2.13.6": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.13.6/scala-compiler-2.13.6-sources.jar", + "integrity": "sha256-YNds7vg1dFKr1kUiTbxhn0MvlxdaxmkhPHwlvX342aQ=", + }, + "2.13.7": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.13.7/scala-compiler-2.13.7-sources.jar", + "integrity": "sha256-gsHXI5FYbPfm5XelBxsHaNF1MW2j7jV3WZwzzrpgw0s=", + }, + "2.13.8": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.13.8/scala-compiler-2.13.8-sources.jar", + "integrity": "sha256-jTTCS05iC+sM5NlspZxFEHtzVnpUasW3WM4o6JFSYwg=", + }, + "2.13.9": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.13.9/scala-compiler-2.13.9-sources.jar", + "integrity": "sha256-aDRGLrBpgx9YqMB4kgwKw303lRBtq98xp8etgm4Ylws=", + }, + "2.13.10": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.13.10/scala-compiler-2.13.10-sources.jar", + "integrity": "sha256-QXzuJo6g4VUoioA/71pipjFtFjtFkV07quO5PlvI3SU=", + }, + "2.13.11": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.13.11/scala-compiler-2.13.11-sources.jar", + "integrity": "sha256-fPsaDG2TngX4ImLJ1z8PqCpmSdBVvwuh9enRDWQIYLs=", + }, + "2.13.12": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.13.12/scala-compiler-2.13.12-sources.jar", + "integrity": "sha256-ZK/GftNZm6fXCwQdk6Hcxcv/ubJgPVLMkqZmFdDZC2Q=", + }, + "2.13.13": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.13.13/scala-compiler-2.13.13-sources.jar", + "integrity": "sha256-zhdtHcdDafnsARq2xipLRdmXl34L0voAbl3MixrJWSE=", + }, + "2.13.14": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.13.14/scala-compiler-2.13.14-sources.jar", + "integrity": "sha256-1vwsYsuQ4y1cCKH70xLixZldKhKPic40pRgeUsv/CAk=", + }, + "2.13.15": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.13.15/scala-compiler-2.13.15-sources.jar", + "integrity": "sha256-EYODcz3jqnFS3IBhfCbcp00b4vL8hbQABz0eLelHZz4=", + }, + "2.13.16": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/2.13.16/scala-compiler-2.13.16-sources.jar", + "integrity": "sha256-6SLcL5fVRf3Ag/L88pFesAx2R/N3C5VIneC8nve1Sj4=", + }, + "3.1.0": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.1.0/scala3-compiler_3-3.1.0-sources.jar", + "integrity": "sha256-tULCgB/phMLaHCKNZIeAHWZy/4LXhx9ghylO6FL5qlw=", + }, + "3.1.1": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.1.1/scala3-compiler_3-3.1.1-sources.jar", + "integrity": "sha256-YLW1zsZwLENfy5ZJBI9bpVMnCxRLtTiAvB/sPw2z2zY=", + }, + "3.1.2": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.1.2/scala3-compiler_3-3.1.2-sources.jar", + "integrity": "sha256-t1OeqlDxa/QKXBCYlJSXLvtUZYCCmQo/5WtsVUUEBVA=", + }, + "3.1.3": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.1.3/scala3-compiler_3-3.1.3-sources.jar", + "integrity": "sha256-7UE3FPJ6UvJTUpinSCTquNbZmzLO0aEnMLp6FUw1H2Q=", + }, + "3.2.0": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.2.0/scala3-compiler_3-3.2.0-sources.jar", + "integrity": "sha256-hRkGR7EzKJSODIC4gRy4VqB/ThXnL22aV+Fr8f5QteE=", + }, + "3.2.1": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.2.1/scala3-compiler_3-3.2.1-sources.jar", + "integrity": "sha256-5w2zKJsMh5gHSlLYue21+EVdU9gK14PbCWxuu1mSTFI=", + }, + "3.2.2": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.2.2/scala3-compiler_3-3.2.2-sources.jar", + "integrity": "sha256-Zp1YD8So08Li0T1XNa6b4F1WdhP+REgt5bzF4uLuieo=", + }, + "3.3.0": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.3.0/scala3-compiler_3-3.3.0-sources.jar", + "integrity": "sha256-c4IrUYMQ6O3vPxqn+hewV7ChatBVOv6Hj54Cn6p4sNY=", + }, + "3.3.1": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.3.1/scala3-compiler_3-3.3.1-sources.jar", + "integrity": "sha256-TIMApZxGtzq7vHX44cGup8kLDhRvHC20+jweIrkMvQc=", + }, + "3.3.2": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.3.2/scala3-compiler_3-3.3.2-sources.jar", + "integrity": "sha256-6kOJFARKj81b7V0IsDw+GrKobpmaKeKIMdh+aOfidjc=", + }, + "3.3.3": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.3.3/scala3-compiler_3-3.3.3-sources.jar", + "integrity": "sha256-fcisUB2zFELAxla+7HpuGzn/kWeErkHaxeFdDaTrsu4=", + }, + "3.3.4": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.3.4/scala3-compiler_3-3.3.4-sources.jar", + "integrity": "sha256-AMilVs9s/Y3Ppkx9qyhbgIKkZntYEh6EptwUSxmPKS0=", + }, + "3.3.5": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.3.5/scala3-compiler_3-3.3.5-sources.jar", + "integrity": "sha256-fAqPm6QvZ6U9rcCzGl2YpTGoiTs8DOqt2lQEEjfyXdA=", + }, + "3.3.6": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.3.6/scala3-compiler_3-3.3.6-sources.jar", + "integrity": "sha256-EuGTpE6VhXO34oJppV31e71dyIYoOlBec3XEr83LSI4=", + }, + "3.4.0": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.4.0/scala3-compiler_3-3.4.0-sources.jar", + "integrity": "sha256-ynvU+5TwQecSZv/ZRKtAXPfyGEFNmT+0IHYegyijMVM=", + }, + "3.4.1": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.4.1/scala3-compiler_3-3.4.1-sources.jar", + "integrity": "sha256-6eCSobLj4PSdD2hiA/r6X6oAg4tz3ct6P/aNHfLQOfo=", + }, + "3.4.2": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.4.2/scala3-compiler_3-3.4.2-sources.jar", + "integrity": "sha256-6IfORj+f5ltzK8Jq8UYrircJfo37xYLKiYI6jMW/OD4=", + }, + "3.4.3": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.4.3/scala3-compiler_3-3.4.3-sources.jar", + "integrity": "sha256-PEE++popIe9Z2n8GXERa4ba5cFfLvGsWlXrQUqV1o84=", + }, + "3.5.0": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.5.0/scala3-compiler_3-3.5.0-sources.jar", + "integrity": "sha256-TN3nkpPFvJ8X+DNvfqcB2QW/pOw8O+1Bod4N3eZ5naY=", + }, + "3.5.1": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.5.1/scala3-compiler_3-3.5.1-sources.jar", + "integrity": "sha256-M1YUfOfltzYm0y4E8+VKKHNhtFIULUhRYQoiMAUco4M=", + }, + "3.5.2": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.5.2/scala3-compiler_3-3.5.2-sources.jar", + "integrity": "sha256-p1D2uTqk2Lc62eb5Lih3JMb6YAeQDC/0kyHiCghHjVc=", + }, + "3.6.0": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.6.0/scala3-compiler_3-3.6.0-sources.jar", + "integrity": "sha256-fMD089tfishxm+uJTGR/WMoECqthafAILRf3hUHQAPE=", + }, + "3.6.1": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.6.1/scala3-compiler_3-3.6.1-sources.jar", + "integrity": "sha256-2unD++OrpXBH2JtFRx9lXu1+WGruVlKDL6x7s/xrFNw=", + }, + "3.6.2": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.6.2/scala3-compiler_3-3.6.2-sources.jar", + "integrity": "sha256-ughbUVD4Ns0KuEEYn55pKfdDRLzlt/75lJVnXqOVFG0=", + }, + "3.6.3": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.6.3/scala3-compiler_3-3.6.3-sources.jar", + "integrity": "sha256-cWwD0D4sZmMRxlJ86JrH57sFkVagRbrkBqSh9Iihbps=", + }, + "3.6.4": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.6.4/scala3-compiler_3-3.6.4-sources.jar", + "integrity": "sha256-bev5Oa1UiAAQ16dCcyBB9e6Q0mLrRcn7k+b2i2V//k4=", + }, + "3.7.0": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.7.0/scala3-compiler_3-3.7.0-sources.jar", + "integrity": "sha256-Xd/wnmEuDHgvXlImK9TJoeDZ//9KSSrA/CjENfquW2Q=", + }, + "3.7.1": { + "url": "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.7.1/scala3-compiler_3-3.7.1-sources.jar", + "integrity": "sha256-C6oxezCyv4+ESER5pl8qPcPhrHv4BEAtmUQ8P+5A/MU=", + }, +} diff --git a/scala/private/macros/scala_repositories.bzl b/scala/private/macros/scala_repositories.bzl index 1dddef76a..590c84cf6 100644 --- a/scala/private/macros/scala_repositories.bzl +++ b/scala/private/macros/scala_repositories.bzl @@ -1,5 +1,12 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@rules_scala_config//:config.bzl", "SCALA_VERSIONS") +load("//:scala_config.bzl", "DEFAULT_SCALA_VERSION") +load( + "//scala/private:macros/compiler_sources_integrity.bzl", + "COMPILER_SOURCES", + "URL_PREFIX", + "URL_SUFFIX_BY_MAJOR_VERSION", +) load( "//scala:scala_cross_version.bzl", "extract_major_version", @@ -50,15 +57,113 @@ compiler_sources_repo = repository_rule( implementation = _compiler_sources_repo_impl, ) +def _get_compiler_srcjar(scala_version, scala_compiler_srcjar): + if scala_compiler_srcjar: + return scala_compiler_srcjar + + compiler_srcjar = COMPILER_SOURCES.get(scala_version, None) + + if compiler_srcjar: + return compiler_srcjar + + # Try to guess what the correct URL will be for scala_version based on its + # major semver number. + requested_major_version = scala_version.split(".", 1)[0] + highest_known_major_version = URL_SUFFIX_BY_MAJOR_VERSION.keys()[-1] + guessed_suffix = ( + URL_SUFFIX_BY_MAJOR_VERSION.get(requested_major_version, None) or + URL_SUFFIX_BY_MAJOR_VERSION[highest_known_major_version] + ) + guessed_compiler_srcjar_url = ( + URL_PREFIX + guessed_suffix.format(version = scala_version) + ) + no_compiler_source_jar_integrity_error = """ +No compiler source jar integrity data exists in rules_scala for Scala version +{version}. + +Please supply a compiler_srcjar object in your MODULE.bazel or legacy WORKSPACE +file. For example, replicating the integrity data for Scala version {default} +in MODULE.bazel would look like: + + scala_deps = use_extension( + "@rules_scala//scala/extensions:deps.bzl", + "scala_deps", + ) + scala_deps.scala() + scala_deps.compiler_srcjar( + integrity = "{integrity}", + url = "{url}", + version = "{default}", + ) + +The equivalent legacy WORKSPACE configuration would look like: + + load("@rules_scala//scala:toolchains.bzl", "scala_toolchains") + + scala_toolchains( + scala_compiler_srcjars = {{ + "{default}": {{ + "integrity": "{integrity}", + "url": "{url}", + }}, + }}, + ) + +Calculate the integrity value using OpenSSL or similar utilities (change the +following URL if it's incorrect for Scala version {version}): + + curl -L {guessed_compiler_srcjar_url} | + openssl dgst -sha256 -binary | openssl base64 + +Then add the "sha256-" prefix manually to produce the final "integrity" value. + +Also: + +- You may replace the single "url" with a list of multiple "urls", or with a + "label" if a BUILD target exists for the source jar. However, you can use only + one of "url", "urls", or "label". + +- You may omit the "integrity" value, or the alternative "sha256" value. + However, Bazel will emit a warning about obtaining a "canonical reproducible + form" by using a specific "integrity" value. You can then add that "integrity" + value to the compiler_srcjar object (if you trust it). + +For more information, see: + +- the Subresource Integrity format description at + https://developer.mozilla.org/docs/Web/Security/Subresource_Integrity + and the formal spec at https://www.w3.org/TR/sri-2/ + +- the docstring from the compiler_srcjar tag_class in + @rules_scala//scala/extensions:deps.bzl + +- the docstring from scala_toolchains() in @rules_scala//scala:toolchains.bzl + +- the dt_patches/test_dt_patches_user_srcjar/{{MODULE.bazel,WORKSPACE}} files + in rules_scala for more examples + +""".format( + version = scala_version, + default = DEFAULT_SCALA_VERSION, + guessed_compiler_srcjar_url = guessed_compiler_srcjar_url, + **COMPILER_SOURCES[DEFAULT_SCALA_VERSION] + ) + fail(no_compiler_source_jar_integrity_error) + def _validate_scalac_srcjar(srcjar): - if type(srcjar) != "dict": - return False - oneof = ["url", "urls", "label"] count = 0 - for key in oneof: - if srcjar.get(key): - count += 1 - return count == 1 + + if type(srcjar) == "dict": + for key in ["url", "urls", "label"]: + if srcjar.get(key): + count += 1 + + if count != 1: + fail( + "scala_compiler_srcjar invalid, must be a dict " + + "with exactly one of \"label\", \"url\" or \"urls\" keys, got: " + + repr(srcjar), + ) def dt_patched_compiler_setup(scala_version, scala_compiler_srcjar = None): scala_major_version = extract_major_version(scala_version) @@ -86,14 +191,9 @@ def dt_patched_compiler_setup(scala_version, scala_compiler_srcjar = None): " srcs=[\"scala/tools/nsc/symtab/SymbolLoaders.scala\"]," if scala_major_version.startswith("2.") else " srcs=[\"dotty/tools/dotc/core/SymbolLoaders.scala\"],", ")", ]) - default_scalac_srcjar = { - "url": "https://repo1.maven.org/maven2/org/scala-lang/scala-compiler/%s/scala-compiler-%s-sources.jar" % (scala_version, scala_version) if scala_major_version.startswith("2.") else "https://repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/%s/scala3-compiler_3-%s-sources.jar" % (scala_version, scala_version), - } - srcjar = scala_compiler_srcjar if scala_compiler_srcjar != None else default_scalac_srcjar - _validate_scalac_srcjar(srcjar) or fail( - ("scala_compiler_srcjar invalid, must be a dict with exactly one of \"label\", \"url\"" + - " or \"urls\" keys, got: ") + repr(srcjar), - ) + srcjar = _get_compiler_srcjar(scala_version, scala_compiler_srcjar) + _validate_scalac_srcjar(srcjar) + if srcjar.get("label"): dt_patched_compiler( name = "scala_compiler_source" + version_suffix(scala_version), diff --git a/scripts/README.md b/scripts/README.md index 89fd88702..654165e95 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -119,8 +119,20 @@ helps avoid suprises when not using `USE_BAZEL_VERSION`. ## [`update_protoc_integrity.py`](./update_protoc_integrity.py) -Updates `protoc/private/protoc_integrity.bzl`. +Updates [`protoc/private/protoc_integrity.bzl`]( +../protoc/private/protoc_integrity.bzl). Upon a new release of [`protocolbuffers/protobuf`](https://github.com/protocolbuffers/protobuf/releases) add the new version to the `PROTOC_VERSIONS` at the top of this file and run it. + +## [`update_compiler_sources_integrity.py`][] + +Updates [`scala/private/macros/compiler_sources_integrity.bzl`]( +../scala/private/macros/compiler_sources_integrity.bzl). + +Upon a new [Scala version release](https://www.scala-lang.org/download/all.html), +add the new version to the `SCALA_VERSIONS` at the top of this file and run it. + +[`update_compiler_sources_integrity.py`]: + ./update_compiler_sources_integrity.py diff --git a/scripts/lib/__init__.py b/scripts/lib/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/scripts/lib/update_integrity.py b/scripts/lib/update_integrity.py new file mode 100644 index 000000000..bed8caccd --- /dev/null +++ b/scripts/lib/update_integrity.py @@ -0,0 +1,155 @@ +"""Utilities for `update_*_integrity.py` scripts.""" + +from base64 import b64encode + +from pathlib import Path + +import argparse +import ast +import hashlib +import json +import re +import sys +import urllib.request + + +class UpdateIntegrityError(Exception): + """Errors raised explicitly by this module.""" + + +def get_integrity_file_path_and_generated_by( + integrity_file_path, + script_path, +): + """Generates the integrity file's absolute path and its docstring. + + Args: + integrity_file_path: integrity file path relative to the repo root + script_path: path to the calling script, presumed to be in //scripts + """ + script_file = Path(script_path) + repo_root = script_file.parent.parent + integrity_file = repo_root / integrity_file_path + return ( + integrity_file, + f'Generated and updated by {script_file.relative_to(repo_root)}.', + ) + + +def get_artifact_integrity(url): + """Emits the integrity string for the specified artifact at `url`. + + Args: + url: URL from which to download the artifact + + Returns: + a string starting with `sha256-` and ending with the base 64 encoded + sha256 checksum of the artifact file + + Raises: + `UpdateIntegrityError` if downloading or checksumming fails + """ + try: + with urllib.request.urlopen(url) as data: + body = data.read() + + sha256 = hashlib.sha256(body).digest() + return f'sha256-{b64encode(sha256).decode('utf-8')}' + + except Exception as err: + msg = f'while processing {url}: {err}' + raise UpdateIntegrityError(msg) from err + + +def stringify_object(data): + """Pretty prints `data` as a Starlark object to emit into an output file. + + Args: + data: a Python list or dict + + Returns: + a pretty-printed string version of `data` to represent a valid Starlark + object in the output file + """ + result = ( + json.dumps(data, indent=4) + .replace('true', 'True') + .replace('false', 'False') + ) + # Add trailing commas. + return re.sub(r'([]}"])\n', r'\1,\n', result) + '\n' + + +def sorted_semver_keyed_dict(semver_keyed_dict, reverse=False): + """Returns a sorted copy of semver_keyed_dict.""" + return dict(sorted( + semver_keyed_dict.items(), + key=lambda item: [int(n) for n in item[0].split(".")], + reverse=reverse, + )) + + +def load_existing_data(existing_file, marker): + """Loads existing integrity data from `existing_file`. + + This enables the script to avoid redownloading artifacts when the integrity + information already exists. + + Args: + existing_file: path to the existing integrity file + marker: string identifying the beginning of the integrity data object + + Returns: + the existing integrity data from `existing_file`, + or `{}` if the file does not exist + """ + if not existing_file.exists(): + return {} + + with existing_file.open('r', encoding='utf-8') as f: + data = f.read() + + start = data.find(marker) + + if start == -1: + msg = f'"{marker}" not found in {existing_file}' + raise UpdateIntegrityError(msg) + + return ast.literal_eval(data[start + len(marker):]) + + +def update_integrity_file(usage, file_path, data_marker, update_data, emit_data): + """Implements `main()` for integrity file updater scripts. + + Args: + usage: command line usage summary line + file_path: path to the integrity file to generate or update + data_marker: line prefix marking the start of the integrity data + update_data: function `(existing data) -> updated data` + emit_data: function `(open file handle, updated data) -> None` + + Raises: + UpdateIntegrityError if any operation fails + """ + parser = argparse.ArgumentParser(description = usage) + + parser.add_argument( + '--integrity_file', + type=str, + default=str(file_path), + help=f'integrity file path (default: {file_path})', + ) + + args = parser.parse_args() + integrity_file = Path(args.integrity_file) + + try: + existing_data = load_existing_data(integrity_file, data_marker) + updated_data = update_data(existing_data) + + with integrity_file.open('w', encoding = 'utf-8') as f: + emit_data(f, updated_data) + + except UpdateIntegrityError as err: + print(f'Failed to update {integrity_file}: {err}', file=sys.stderr) + sys.exit(1) diff --git a/scripts/update_compiler_sources_integrity.py b/scripts/update_compiler_sources_integrity.py new file mode 100755 index 000000000..f5eaba951 --- /dev/null +++ b/scripts/update_compiler_sources_integrity.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Updates `scala/private/macros/compiler_sources_integrity.bzl`. + +`compiler_sources_integrity.bzl` contains the mapping from Scala versions to +their source URLs and integrity hashes. + +Only computes the integrity information for compiler versions that don't already +exist in the integrity file. +""" + +from lib.update_integrity import ( + get_artifact_integrity, + get_integrity_file_path_and_generated_by, + sorted_semver_keyed_dict, + stringify_object, + update_integrity_file, +) + + +# These are matched with the versions from //dt_patches:dt_patch_test.sh. +SCALA_VERSIONS = [ + "2.11.12", +] + [ + f'2.12.{patch}' for patch in range(1, 21) # 2.12.1 to 2.12.20 +] + [ + f'2.13.{patch}' for patch in range(0, 17) # 2.13.0 to 2.13.16 +] + [ + f'3.1.{patch}' for patch in range(0, 4) # 3.1.0 to 3.1.3 +] + [ + f'3.2.{patch}' for patch in range(0, 3) # 3.2.0 to 3.2.2 +] + [ + f'3.3.{patch}' for patch in range(0, 7) # 3.3.0 to 3.3.6 +] + [ + f'3.4.{patch}' for patch in range(0, 4) # 3.4.0 to 3.4.3 +] + [ + f'3.5.{patch}' for patch in range(0, 3) # 3.5.0 to 3.5.2 +] + [ + f'3.6.{patch}' for patch in range(0, 5) # 3.6.0 to 3.6.4 +] + [ + f'3.7.{patch}' for patch in range(0, 2) # 3.7.0 to 3.7.1 +] + +DATA_MARKER = "COMPILER_SOURCES = " +URL_PREFIX = "https://repo1.maven.org/maven2/org/scala-lang/" +URL_SUFFIX_BY_MAJOR_VERSION = { + "2": "scala-compiler/{version}/scala-compiler-{version}-sources.jar", + "3": "scala3-compiler_3/{version}/scala3-compiler_3-{version}-sources.jar", +} + +INTEGRITY_FILE, GENERATED_BY = get_integrity_file_path_and_generated_by( + 'scala/private/macros/compiler_sources_integrity.bzl', + __file__, +) +INTEGRITY_FILE_HEADER = f'''"""Scala compiler source JAR integrity metadata. + +{GENERATED_BY} +""" + +URL_PREFIX = "{URL_PREFIX}" +URL_SUFFIX_BY_MAJOR_VERSION = ''' + + +class UpdateCompilerSourcesIntegrityError(Exception): + """Errors raised explicitly by this module.""" + + +def get_compiler_source_integrity(scala_version): + """Generates the URL and integrity value for a specific Scala version. + + Args: + scala_version: the scala version for which to generate a URL and + integrity value + + Returns: + a `{"url", "integrity"}` dict for the `scala_version` compiler sources + """ + major_version = scala_version.split(".", 1)[0] + url_suffix = URL_SUFFIX_BY_MAJOR_VERSION.get(major_version, None) + + if url_suffix is None: + msg = "unknown major Scala version: " + scala_version + raise UpdateCompilerSourcesIntegrityError(msg) + + url = URL_PREFIX + url_suffix.format(version = scala_version) + print(f'Generating integrity for:\n {url}') + return {"url": url, "integrity": get_artifact_integrity(url)} + + +def update_compiler_sources_integrity_data(existing_data): + """Generates or updates compiler sources integrity data. + + Does not generate new compiler source integrity data for Scala versions + already in `existing_data`. + + Args: + existing_data: existing compiler source integrity data + + Returns: + a new `{scala version: integrity data}` dict combining existing and new + compiler sources integrity data + """ + updated_data = existing_data | { + version: get_compiler_source_integrity(version) + for version in SCALA_VERSIONS + if version not in existing_data + } + return sorted_semver_keyed_dict(updated_data) + + +def emit_compiler_sources_integrity_data(output_file, integrity_data): + """Writes the updated compiler_sources integrity data to the `output_file`. + + Args: + output_file: open file object for the updated compiler sources integrity + file + integrity_data: compiler sources integrity data to emit into + `output_file` + """ + output_file.write(INTEGRITY_FILE_HEADER) + output_file.write(stringify_object(URL_SUFFIX_BY_MAJOR_VERSION)) + output_file.write(DATA_MARKER) + output_file.write(stringify_object(integrity_data)) + + +if __name__ == "__main__": + update_integrity_file( + "Updates Scala compiler source JAR integrity information.", + INTEGRITY_FILE, + DATA_MARKER, + update_compiler_sources_integrity_data, + emit_compiler_sources_integrity_data, + ) diff --git a/scripts/update_protoc_integrity.py b/scripts/update_protoc_integrity.py index e7a5e698d..b5ca1a9a0 100755 --- a/scripts/update_protoc_integrity.py +++ b/scripts/update_protoc_integrity.py @@ -9,22 +9,16 @@ Only computes integrity information for a `protoc` distribution if it doesn't already exist in the integrity file. - -This borrows some code from `scripts/create_repository.py` that could probably -be extracted into a common module. Specifically, `emit_protoc_integrity_file()` -borrows heavily from `ArtifactUpdater.write_to_file()`. """ -from base64 import b64encode -from pathlib import Path +from lib.update_integrity import ( + get_artifact_integrity, + get_integrity_file_path_and_generated_by, + sorted_semver_keyed_dict, + stringify_object, + update_integrity_file, +) -import argparse -import ast -import hashlib -import json -import re -import urllib.request -import sys PROTOC_VERSIONS = [ "31.1", @@ -81,12 +75,14 @@ ], } -THIS_FILE = Path(__file__) -REPO_ROOT = THIS_FILE.parent.parent -INTEGRITY_FILE = REPO_ROOT / 'protoc/private/protoc_integrity.bzl' +DATA_MARKER = "PROTOC_BUILDS = " +INTEGRITY_FILE, GENERATED_BY = get_integrity_file_path_and_generated_by( + 'protoc/private/protoc_integrity.bzl', + __file__, +) INTEGRITY_FILE_HEADER = f'''"""Protocol compiler build and integrity metadata. -Generated and updated by {THIS_FILE.relative_to(REPO_ROOT)}. +{GENERATED_BY} """ PROTOC_RELEASES_URL = "{PROTOC_RELEASES_URL}" @@ -95,11 +91,7 @@ "{PROTOC_DOWNLOAD_SUFFIX}" ) -''' - - -class UpdateProtocIntegrityError(Exception): - """Errors raised explicitly by this module.""" +PROTOC_VERSIONS = ''' def get_protoc_integrity(platform, version): @@ -117,21 +109,11 @@ def get_protoc_integrity(platform, version): sha256 checksum of the `protoc` distribution file Raises: - `UpdateProtocIntegrityError` if downloading or checksumming fails + `UpdateIntegrityError` if downloading or checksumming fails """ url = PROTOC_DOWNLOAD_URL.format(version = version, platform = platform) print(f'Updating protoc {version} for {platform}:\n {url}') - - try: - with urllib.request.urlopen(url) as data: - body = data.read() - - sha256 = hashlib.sha256(body).digest() - return f'sha256-{b64encode(sha256).decode('utf-8')}' - - except Exception as err: - msg = f'while processing {url}: {err}' - raise UpdateProtocIntegrityError(msg) from err + return get_artifact_integrity(url) def add_build_data(platform, exec_compat, existing_build): @@ -154,96 +136,52 @@ def add_build_data(platform, exec_compat, existing_build): return { "exec_compat": exec_compat, - "integrity": dict(sorted(integrity.items(), reverse=True)), + "integrity": sorted_semver_keyed_dict(integrity, reverse=True), } -def stringify_object(data): - """Pretty prints `data` as a Starlark object to emit into the output file. +def update_protoc_integrity_data(existing_data): + """Generates or updates `protoc` integrity data. + + Does not generate new `protoc` integrity data for versions and builds + already in `existing_data`. Args: - data: a Python list or dict + existing_data: existing `protoc` integrity data Returns: - a pretty-printed string version of `data` to represent a valid Starlark - object in the output file + a new `{protobuf version: integrity data}` dict combining existing and + new `protoc` integrity data """ - result = ( - json.dumps(data, indent=4) - .replace('true', 'True') - .replace('false', 'False') - ) - # Add trailing commas. - return re.sub(r'([]}"])\n', r'\1,\n', result) + '\n' + updated_data = { + platform: add_build_data( + platform, + exec_compat, + existing_data.get(platform, {}), + ) + for platform, exec_compat in PROTOC_BUILDS.items() + } + return dict(sorted(updated_data.items())) -def emit_protoc_integrity_file(output_file, integrity_data): +def emit_protoc_integrity_data(output_file, integrity_data): """Writes the updated `protoc` integrity data to the `output_file`. Args: - output_file: path to the updated `protoc` integrity file + output_file: open file object for the updated `protoc` integrity file integrity_data: `protoc` integrity data to emit into `output_file` """ - with output_file.open('w', encoding = 'utf-8') as data: - data.write(INTEGRITY_FILE_HEADER) - data.write("PROTOC_VERSIONS = ") - data.write(stringify_object(PROTOC_VERSIONS)) - data.write("\nPROTOC_BUILDS = ") - data.write(stringify_object(dict(sorted(integrity_data.items())))) - - -def load_existing_data(existing_file): - """Loads existing `protoc` integrity data from `existing_file`. - - This enables the script to avoid redownloading `protoc` distribution files - when the integrity information already exists. - - Args: - existing_file: path to the existing integrity file - - Returns: - the existing `PROTOC_BUILDS` integrity data from `existing_file`, - or `{}` if the file does not exist - """ - if not existing_file.exists(): - return {} - - with existing_file.open('r', encoding='utf-8') as f: - data = f.read() - - marker = 'PROTOC_BUILDS = ' - start = data.find(marker) - - if start == -1: - msg = f'"{marker}" not found in {existing_file}' - raise UpdateProtocIntegrityError(msg) - - return ast.literal_eval(data[start + len(marker):]) + output_file.write(INTEGRITY_FILE_HEADER) + output_file.write(stringify_object(PROTOC_VERSIONS)) + output_file.write(DATA_MARKER) + output_file.write(stringify_object(integrity_data)) if __name__ == "__main__": - parser = argparse.ArgumentParser( - description = "Updates precompiled `protoc` distribution information.", - ) - - parser.add_argument( - '--integrity_file', - type=str, - default=str(INTEGRITY_FILE), - help=f'`protoc` integrity file path (default: {INTEGRITY_FILE})', + update_integrity_file( + "Updates precompiled `protoc` distribution information.", + INTEGRITY_FILE, + DATA_MARKER, + update_protoc_integrity_data, + emit_protoc_integrity_data, ) - - args = parser.parse_args() - integrity_file = Path(args.integrity_file) - - try: - existing_data = load_existing_data(integrity_file) - updated_data = { - k: add_build_data(k, v, existing_data.get(k, {})) - for k, v in PROTOC_BUILDS.items() - } - emit_protoc_integrity_file(integrity_file, updated_data) - - except UpdateProtocIntegrityError as err: - print(f'Failed to update {integrity_file}: {err}', file=sys.stderr) - sys.exit(1) diff --git a/test/compiler_sources_integrity/.bazelrc b/test/compiler_sources_integrity/.bazelrc new file mode 100644 index 000000000..005efba2f --- /dev/null +++ b/test/compiler_sources_integrity/.bazelrc @@ -0,0 +1 @@ +import ../../.bazelrc diff --git a/test/compiler_sources_integrity/.bazelversion b/test/compiler_sources_integrity/.bazelversion new file mode 100644 index 000000000..e8be68404 --- /dev/null +++ b/test/compiler_sources_integrity/.bazelversion @@ -0,0 +1 @@ +7.6.1 diff --git a/test/compiler_sources_integrity/BUILD b/test/compiler_sources_integrity/BUILD new file mode 100644 index 000000000..c762510b3 --- /dev/null +++ b/test/compiler_sources_integrity/BUILD @@ -0,0 +1,4 @@ +alias( + name = "src", + actual = "@scala_compiler_sources//:src", +) diff --git a/test/compiler_sources_integrity/MODULE.bazel b/test/compiler_sources_integrity/MODULE.bazel new file mode 100644 index 000000000..c0ba183e2 --- /dev/null +++ b/test/compiler_sources_integrity/MODULE.bazel @@ -0,0 +1,45 @@ +"""Bazel module for test/shell/test_compiler_sources_integrity.sh""" + +module(name = "test_compiler_sources_integrity") + +bazel_dep(name = "rules_scala") +local_path_override( + module_name = "rules_scala", + path = "../..", +) + +scala_config = use_extension( + "@rules_scala//scala/extensions:config.bzl", + "scala_config", +) +scala_config.settings( + # Most of the time, this will trigger compiler source downloads, not a + # direct dependency on @scala_compiler_sources. The `alias` from //:src to + # @scala_compiler_sources//:src isn't common, but makes the test very fast. + enable_compiler_dependency_tracking = True, +) + +scala_deps = use_extension( + "@rules_scala//scala/extensions:deps.bzl", + "scala_deps", +) +scala_deps.scala() +scala_deps.settings( + # Since we're using a bogus Scala version in the compiler_srcjar. + validate_scala_version = False, +) + +# Used for test_emit_no_canonical_reproducible_form_warning_for_user_srcjar. +# See that test case's comments for details. +scala_deps.compiler_srcjar( + integrity = "sha256-7UE3FPJ6UvJTUpinSCTquNbZmzLO0aEnMLp6FUw1H2Q=", + url = ( + "https://repo1.maven.org/maven2/org/scala-lang/" + + "scala3-compiler_3/3.1.3/scala3-compiler_3-3.1.3-sources.jar" + ), + version = "3.1.999", +) +use_repo( + scala_deps, + "scala_compiler_sources", +) diff --git a/test/compiler_sources_integrity/WORKSPACE b/test/compiler_sources_integrity/WORKSPACE new file mode 100644 index 000000000..f942f7a49 --- /dev/null +++ b/test/compiler_sources_integrity/WORKSPACE @@ -0,0 +1,57 @@ +workspace(name = "rules_scala_test") + +local_repository( + name = "rules_scala", + path = "../../", +) + +load("@rules_scala//scala:latest_deps.bzl", "rules_scala_dependencies") + +rules_scala_dependencies() + +load("@rules_java//java:rules_java_deps.bzl", "rules_java_dependencies") + +rules_java_dependencies() + +load("@bazel_features//:deps.bzl", "bazel_features_deps") + +bazel_features_deps() + +load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace") + +bazel_skylib_workspace() + +load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") + +protobuf_deps() + +load("@rules_scala//:scala_config.bzl", "scala_config") + +scala_config( + # Most of the time, this will trigger compiler source downloads, not a + # direct dependency on @scala_compiler_sources. The `alias` from //:src to + # @scala_compiler_sources//:src isn't common, but makes the test very fast. + enable_compiler_dependency_tracking = True, +) + +load( + "@rules_scala//scala:toolchains.bzl", + "scala_register_toolchains", + "scala_toolchains", +) + +scala_toolchains( + scala_compiler_srcjars = { + "3.1.999": { + "integrity": "sha256-7UE3FPJ6UvJTUpinSCTquNbZmzLO0aEnMLp6FUw1H2Q=", + "url": ( + "https://repo1.maven.org/maven2/org/scala-lang/" + + "scala3-compiler_3/3.1.3/scala3-compiler_3-3.1.3-sources.jar" + ), + }, + }, + # Since we're using a bogus Scala version in the compiler_srcjar. + validate_scala_version = False, +) + +scala_register_toolchains() diff --git a/test/compiler_sources_integrity/protobuf.patch b/test/compiler_sources_integrity/protobuf.patch new file mode 120000 index 000000000..c8f00be6e --- /dev/null +++ b/test/compiler_sources_integrity/protobuf.patch @@ -0,0 +1 @@ +../../protoc/0001-protobuf-19679-rm-protoc-dep.patch \ No newline at end of file diff --git a/test/shell/test_compiler_sources_integrity.sh b/test/shell/test_compiler_sources_integrity.sh new file mode 100755 index 000000000..7c264d3e3 --- /dev/null +++ b/test/shell/test_compiler_sources_integrity.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash +# +# Reproduces and validates the fix for bazel-contrib/rules_scala#1743. +# +# Specifically, we must not emit "canonical reproducible form" warnings for +# known Scala versions when enabling dependency tracking via: +# +# scala_config.settings( +# enable_compiler_dependency_tracking = True, +# ) +# +# Instead, we fail the build with a message describing how to use +# scala_deps.compiler_srcjar to provide an "integrity" value. +# +# See: +# +# - _get_compiler_srcjar() from scala/private/macros/scala_repositories.bzl +# - scala/private/macros/compiler_sources_integrity.bzl +# - scripts/update_compiler_sources_integrity.py + +set -euo pipefail + +dir="$( cd "${BASH_SOURCE[0]%/*}" && echo "${PWD%/test/shell}" )" +test_source="${dir}/test/shell/${BASH_SOURCE[0]#*test/shell/}" +# shellcheck source=./test_runner.sh +. "${dir}"/test/shell/test_runner.sh +. "${dir}"/test/shell/test_helper.sh +export USE_BAZEL_VERSION=${USE_BAZEL_VERSION:-$(cat $dir/.bazelversion)} + +# Setup and teardown + +_clean() { + bazel clean --expunge_async >/dev/null 2>&1 +} + +setup_suite() { + local output + + original_dir="$PWD" + cd "${dir}/test/compiler_sources_integrity" + + local bk_bazel_rc="${dir}/tools/bazel.rc" + + if [[ -f "$bk_bazel_rc" ]]; then + # test_rules_scala_jdk21 from .bazelci/presubmit.yml needs this. + mkdir tools + cp "${bk_bazel_rc}" tools/ + fi + + # The behavior we're testing must not rely on repos generated during previous + # builds or test runs. + _clean +} + +teardown_suite() { + _clean + cd "$original_dir" +} + +# Helpers and assertions + +_REPO_PREFIX='https://repo1.maven.org/maven2/org/scala-lang' + +_scala_2_url() { + local scala_version="$1" + printf '%s/scala-compiler/%s/scala-compiler-%s-sources.jar' \ + "$_REPO_PREFIX" "$scala_version" "$scala_version" +} + +_scala_3_url() { + local scala_version="$1" + printf '%s/scala3-compiler_3/%s/scala3-compiler_3-%s-sources.jar' \ + "$_REPO_PREFIX" "$scala_version" "$scala_version" +} + +_build_with_scala_version() { + local scala_version="${1:-}" + local build_args=() + + if [[ -n "$scala_version" ]]; then + build_args+=("--repo_env=SCALA_VERSION=${scala_version}") + fi + + # Because the macOS BuildKite runner apparently uses an older Bash, leading to + # `build_args[@]` being unbound when empty: + # - https://stackoverflow.com/a/7577209 + bazel build ${build_args[@]+"${build_args[@]}"} //... 2>&1 +} + +_expect_success_without_canonical_reproducible_warning() { + local crf_warning="canonical reproducible form" + local output + + if ! output="$(_build_with_scala_version "$@")"; then + echo "$output" + fail " build failed" + elif [[ "$output" =~ $crf_warning ]]; then + echo "$output" + fail " build output contained \"${crf_warning}\" warning" + elif verbose_test_output; then + echo "$output" + echo -e "${GREEN} \"bazel $*\" output didn't contain \"${crf_warning}\".$NC" + fi +} + +_FAILED_MSG='No compiler source jar integrity data exists' + +_expect_failure_with_guessed_url() { + local scala_version="$1" + local guessed_url="$2" + local output + + if output="$(_build_with_scala_version "$scala_version")"; then + echo "$output" + fail " SCALA_VERSION=${scala_version} build didn't fail" + elif [[ ! "$output" =~ $_FAILED_MSG ]]; then + echo "$output" + fail " error message didn't contain \"${_FAILED_MSG}\"" + elif [[ ! "$output" =~ $guessed_url ]]; then + echo "$output" + fail " error message didn't contain \"${guessed_url}\"" + elif verbose_test_output; then + echo "$output" + echo -e "${GREEN} error message contained \"${_FAILED_MSG}\"" + echo -e " and \"${guessed_url}\".$NC" + fi +} + +# Test cases + +test_emit_no_canonical_reproducible_form_warning_for_default_version() { + _expect_success_without_canonical_reproducible_warning +} + +test_emit_no_canonical_reproducible_form_warning_for_latest_versions() { + local scala_version_pattern='^scala_version = "([0-9.]+)"$' + local f + local line + local version + local versions=() + + for f in "${dir}"/third_party/repositories/scala_*.bzl; do + while IFS= read -r line; do + if [[ "$line" =~ $scala_version_pattern ]]; then + versions+=("${BASH_REMATCH[1]}") + break + fi + done <"$f" + done + + for version in "${versions[@]}"; do + _expect_success_without_canonical_reproducible_warning "$version" + done +} + +test_emit_no_canonical_reproducible_form_warning_for_user_srcjar() { + # Uses a bogus version not in compiler_sources_integrity.bzl, so we know the + # build's relying on the user defined compiler_srcjar. That compiler_srcjar + # instance uses a real URL to ensure the build succeeds for this test case. + _expect_success_without_canonical_reproducible_warning "3.1.999" +} + +test_fail_if_missing_compiler_source_integrity() { + # A value not in scala/private/macros/compiler_sources_integrity.bzl nor + # configured as a compiler_srcjar in MODULE.bazel or WORKSPACE. + local scala_version='2.13.999' + local guessed_url="$(_scala_2_url "$scala_version")" + + _expect_failure_with_guessed_url "$scala_version" "$guessed_url" +} + +test_fail_with_scala3_compiler_source_link() { + # A value not in scala/private/macros/compiler_sources_integrity.bzl nor + # configured as a compiler_srcjar in MODULE.bazel or WORKSPACE. + local scala_version='3.7.999' + local guessed_url="$(_scala_3_url "$scala_version")" + + _expect_failure_with_guessed_url "$scala_version" "$guessed_url" +} + +test_fail_with_scala3_compiler_source_link_for_unknown_major_version() { + # A value not in scala/private/macros/compiler_sources_integrity.bzl that's + # beyond the current highest major version. For now we sort of fake it with a + # Scala 3 URL, but if/when Scala 4 launches, this will prompt us to update it. + local scala_version='4.0.999' + local guessed_url="$(_scala_3_url "$scala_version")" + + _expect_failure_with_guessed_url "$scala_version" "$guessed_url" +} + +# main() + +setup_suite +run_tests "$test_source" "$(get_test_runner "${1:-local}")" +teardown_suite diff --git a/test/shell/test_runner.sh b/test/shell/test_runner.sh index 914b12ea9..ba69af610 100644 --- a/test/shell/test_runner.sh +++ b/test/shell/test_runner.sh @@ -222,7 +222,7 @@ run_tests() { local runner="$2" while IFS= read -r line; do - if [[ "$line" =~ ^_?(test_[A-Za-z0-9_]+)\(\)\ ?\{$ ]]; then + if [[ "$line" =~ ^(_?test_[A-Za-z0-9_]+)\(\)\ ?\{$ ]]; then "$runner" "${BASH_REMATCH[1]}" fi done <"$test_source" diff --git a/test_rules_scala.sh b/test_rules_scala.sh index 77b51f4b6..0b1ced9a4 100755 --- a/test_rules_scala.sh +++ b/test_rules_scala.sh @@ -29,6 +29,7 @@ $runner bazel test "$test_output_flag" //test/... --extra_toolchains="//test_exp $runner bazel build test:ScalaBinaryInGenrule --nolegacy_external_runfiles $runner bazel build //test_statsfile:Simple_statsfile $runner bazel build //test_statsfile:SimpleNoStatsFile_statsfile --extra_toolchains="//test/toolchains:enable_stats_file_disabled_toolchain" +. "${test_dir}"/test_compiler_sources_integrity.sh . "${test_dir}"/test_build_event_protocol.sh . "${test_dir}"/test_compilation.sh . "${test_dir}"/test_deps.sh