From 2a0168b1e6354f1d04be852a91a6fc2e857ee16a Mon Sep 17 00:00:00 2001 From: dandds Date: Fri, 9 Aug 2019 08:55:51 -0400 Subject: [PATCH] Scripts for finding accidental secrets in the repo. This adds the following: - A detect-secrets dependency and a related script (`script/detect_secrets`) to find and alert developers to secrets added to the code. By default, the script will search staged and new, unstaged files. It can optionally search only staged files. - A whitelist, `.secrets.baseline`, that tracks instances of secrets or false positives already in the repo. - Modifies `script/test` to detect secrets as part of the test suite. - Updates to the README regarding the use of detect-secrets. --- .secrets.baseline | 187 ++++++++++++++++++++++++++++++++++++++++++ Pipfile | 1 + Pipfile.lock | 51 ++++++++++-- README.md | 35 ++++++++ script/detect_secrets | 43 ++++++++++ script/test | 3 + 6 files changed, 313 insertions(+), 7 deletions(-) create mode 100644 .secrets.baseline create mode 100755 script/detect_secrets diff --git a/.secrets.baseline b/.secrets.baseline new file mode 100644 index 00000000..8926613d --- /dev/null +++ b/.secrets.baseline @@ -0,0 +1,187 @@ +{ + "exclude": { + "files": null, + "lines": null + }, + "generated_at": "2019-08-13T09:59:21Z", + "plugins_used": [ + { + "base64_limit": 4.5, + "name": "Base64HighEntropyString" + }, + { + "name": "BasicAuthDetector" + }, + { + "hex_limit": 3, + "name": "HexHighEntropyString" + }, + { + "name": "KeywordDetector" + }, + { + "name": "PrivateKeyDetector" + } + ], + "results": { + "Pipfile.lock": [ + { + "hashed_secret": "a355e3e3983231194cfc5981d48ea7ff9a7f9cb6", + "is_secret": false, + "is_verified": false, + "line_number": 4, + "type": "Hex High Entropy String" + } + ], + "README.md": [ + { + "hashed_secret": "d141ce86b0584abb29ee7c24af9afb1e3d871f04", + "is_secret": false, + "is_verified": false, + "line_number": 145, + "type": "Secret Keyword" + } + ], + "alembic.ini": [ + { + "hashed_secret": "9d4e1e23bd5b727046a9e3b4b7db57bd8d6ee684", + "is_secret": false, + "is_verified": false, + "line_number": 38, + "type": "Basic Auth Credentials" + } + ], + "alembic/versions/e0c6eb21771f_reset_migrations_with_new_schema.py": [ + { + "hashed_secret": "999a22300a564f9d2bdca555c2170465fd760ae3", + "is_secret": false, + "is_verified": false, + "line_number": 13, + "type": "Hex High Entropy String" + } + ], + "atst.ini.example": [ + { + "hashed_secret": "abcdb568713c255c81376829da20004ba9463fd3", + "is_secret": false, + "is_verified": false, + "line_number": 2, + "type": "Secret Keyword" + } + ], + "config/base.ini": [ + { + "hashed_secret": "aa419433d95be86df254d499243bee1d5173f1ae", + "is_secret": false, + "is_verified": false, + "line_number": 5, + "type": "Secret Keyword" + }, + { + "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3", + "is_secret": false, + "is_verified": false, + "line_number": 17, + "type": "Secret Keyword" + }, + { + "hashed_secret": "abcdb568713c255c81376829da20004ba9463fd3", + "is_secret": false, + "is_verified": false, + "line_number": 23, + "type": "Secret Keyword" + } + ], + "ssl/certificate-authority/ca.key": [ + { + "hashed_secret": "be4fc4886bd949b369d5e092eb87494f12e57e5b", + "is_secret": false, + "is_verified": false, + "line_number": 1, + "type": "Private Key" + } + ], + "ssl/client-certs/atat.mil.key": [ + { + "hashed_secret": "be4fc4886bd949b369d5e092eb87494f12e57e5b", + "is_secret": false, + "is_verified": false, + "line_number": 1, + "type": "Private Key" + } + ], + "ssl/client-certs/bad-atat.mil.key": [ + { + "hashed_secret": "be4fc4886bd949b369d5e092eb87494f12e57e5b", + "is_secret": false, + "is_verified": false, + "line_number": 1, + "type": "Private Key" + } + ], + "ssl/client-certs/client-ca.key": [ + { + "hashed_secret": "be4fc4886bd949b369d5e092eb87494f12e57e5b", + "is_secret": false, + "is_verified": false, + "line_number": 1, + "type": "Private Key" + } + ], + "ssl/server-certs/dev.cac.atat.codes.key": [ + { + "hashed_secret": "be4fc4886bd949b369d5e092eb87494f12e57e5b", + "is_secret": false, + "is_verified": false, + "line_number": 1, + "type": "Private Key" + } + ], + "tests/acceptance/conftest.py": [ + { + "hashed_secret": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f", + "is_secret": false, + "is_verified": false, + "line_number": 48, + "type": "Basic Auth Credentials" + } + ], + "tests/fixtures/chain/make-chain.sh": [ + { + "hashed_secret": "bad2e396920ce37fe53fc291f90b130d915375fb", + "is_secret": false, + "is_verified": false, + "line_number": 35, + "type": "Secret Keyword" + } + ], + "tests/forms/test_validators.py": [ + { + "hashed_secret": "260408f687da9094705a841acda9b029563053ee", + "is_secret": false, + "is_verified": false, + "line_number": 30, + "type": "Hex High Entropy String" + } + ], + "tests/routes/task_orders/test_new.py": [ + { + "hashed_secret": "e4f14805dfd1e6af030359090c535e149e6b4207", + "is_secret": false, + "is_verified": false, + "line_number": 30, + "type": "Hex High Entropy String" + } + ], + "tests/test_access.py": [ + { + "hashed_secret": "e4f14805dfd1e6af030359090c535e149e6b4207", + "is_secret": false, + "is_verified": false, + "line_number": 481, + "type": "Hex High Entropy String" + } + ] + }, + "version": "0.12.5" +} diff --git a/Pipfile b/Pipfile index 4e169d3a..233fa6c0 100644 --- a/Pipfile +++ b/Pipfile @@ -43,6 +43,7 @@ selenium = "*" honcho = "*" blinker = "*" pytest-mock = "*" +detect-secrets = "*" [requires] python_version = "3.7.3" diff --git a/Pipfile.lock b/Pipfile.lock index 6fe43d84..c9b3b9dd 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "d697cd7c279a761283ea2ffefab08d5854e4c0341b5646839b269ab0d999bf87" + "sha256": "f09d90a1b4b86eff2e0ed453ceefcce4f19d54c57fb50ebed1f02aae8ab83c2a" }, "pipfile-spec": 6, "requires": { @@ -71,18 +71,18 @@ }, "boto3": { "hashes": [ - "sha256:666f37c5852f71925494fc2103b189deafe6702c1d9ae60bead5b1b6466de857", - "sha256:7b77b507221ec15550b02d492804166bcc61ef3a81312968065515a76aa1791b" + "sha256:3ec5b520dbd0a430cdd581a8250991fb0f21ee7e668a8928f15006b312fa86dc", + "sha256:8aec0247131a0db1e33d28ad13910e01e6dfa208e8ab8ee5a4095e92dbaabf45" ], "index": "pypi", - "version": "==1.9.202" + "version": "==1.9.204" }, "botocore": { "hashes": [ - "sha256:71ca578701e746fe947c098e5dee06128d0f6ba98217ba7e29aff0dab8caf82f", - "sha256:e55003c46e71396a551d4b70f39286f8fc4094ac6cf90f5db8d7a68bb4af1f9d" + "sha256:56cd1114e0ce35733e890b321160c8c438243f4fa54d3d074dfa6bdce4ee55aa", + "sha256:f86504bcc9c44d5b2e7b019f2f279b70f17b1400d2fc4775bc009ec473530cad" ], - "version": "==1.12.202" + "version": "==1.12.204" }, "certifi": { "hashes": [ @@ -562,6 +562,20 @@ "index": "pypi", "version": "==1.4" }, + "certifi": { + "hashes": [ + "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939", + "sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695" + ], + "version": "==2019.6.16" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, "click": { "hashes": [ "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", @@ -615,6 +629,14 @@ ], "version": "==4.4.0" }, + "detect-secrets": { + "hashes": [ + "sha256:33593cbf7099ced595e7371fcee31c815a237419de2869d571bbdb651a125d4c", + "sha256:7b8e6d521f5a164ff17dbe3d2691eb85f842133d4e6bb7a23eeb461a0ab4e215" + ], + "index": "pypi", + "version": "==0.12.5" + }, "docopt": { "hashes": [ "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" @@ -666,6 +688,13 @@ "index": "pypi", "version": "==1.0.1" }, + "idna": { + "hashes": [ + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + ], + "version": "==2.8" + }, "importlib-metadata": { "hashes": [ "sha256:23d3d873e008a513952355379d93cbcab874c58f4f034ff657c7a87422fa64e8", @@ -945,6 +974,14 @@ "index": "pypi", "version": "==5.1.2" }, + "requests": { + "hashes": [ + "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", + "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" + ], + "index": "pypi", + "version": "==2.22.0" + }, "selenium": { "hashes": [ "sha256:7491b5391f29a74774d475456d3a138e00fae0b3966f68a100f1f3ad331ce166", diff --git a/README.md b/README.md index fb3f7c38..128f157d 100644 --- a/README.md +++ b/README.md @@ -256,3 +256,38 @@ location /login-dev { ``` The location block will require the same proxy pass configuration as other location blocks for the app. + +## Secrets Detection + +This project uses [detect-secrets](https://github.com/Yelp/detect-secrets) to help prevent secrets from being checked into source control. Secret detection is run automatically as part of `script/test` and can be run separately with `script/detect_secrets`. + +If you need to check in a file that raises false positives from `detect-secrets`, you can add it to the whitelist. Run: + +``` +pipenv run detect-secrets scan --update .secrets.baseline +``` + +and then: + +``` +pipenv run detect-secrets audit .secrets.baseline +``` + +The audit will open an interactive prompt where you can whitelist the file. This is useful if you're checking in an entire file that looks like or is a secret (like a sample PKI file). + +Alternatively, you can add a `# pragma: allowlist secret` comment to the line that raised the false positive. See the [detect-secret](https://github.com/Yelp/detect-secrets#inline-allowlisting) docs for more information. + +It's recommended that you add a pre-commit hook to invoke `script/detect_secrets`. Add the example below or something equivalent to `.git/hooks/pre-commit`: + +``` +#!/usr/bin/env bash + +if ./script/detect_secrets staged; then + echo "secrets check passed" +else + echo -e "**SECRETS DETECTED**" + exit 1 +fi +``` + +Also note that if the line number of a previously whitelisted secret changes, the whitelist file, `.secrets.baseline`, will be updated and needs to be committed. diff --git a/script/detect_secrets b/script/detect_secrets new file mode 100755 index 00000000..1322528b --- /dev/null +++ b/script/detect_secrets @@ -0,0 +1,43 @@ +#! .venv/bin/python +import subprocess +import sys + +from detect_secrets.pre_commit_hook import main as find_secrets + + +TRACKED_CHANGES = ["git", "diff", "HEAD", "--name-only"] +STAGED_CHANGES = ["git", "diff", "--cached", "--name-only"] +UNTRACKED_CHANGES = ["git", "ls-files", "--others", "--exclude-standard"] + + +def git_file_list(cmd): + comproc = subprocess.run(cmd, capture_output=True) + return [f.decode() for f in comproc.stdout.split()] + + +def git_staged_files(): + return git_file_list(STAGED_CHANGES) + + +def git_all_files(): + return git_file_list(TRACKED_CHANGES) + git_file_list(UNTRACKED_CHANGES) + + +def main(arg): + """ + If `arg` is "staged", this will only check files that have been + staged to the git index. Otherwise, it will check staged and + unstaged files. + """ + files = [] + if arg == "staged": + files = git_staged_files() + else: + files = git_all_files() + + return find_secrets(["--baseline", ".secrets.baseline"] + files) + + +if __name__ == "__main__": + arg = sys.argv[1] if len(sys.argv) > 1 else None + sys.exit(main(arg)) diff --git a/script/test b/script/test index a97034b6..fa95fa6b 100755 --- a/script/test +++ b/script/test @@ -24,5 +24,8 @@ RUN_JS_TESTS="true" # Check python formatting source ./script/format check +# Check for secrets +./script/detect_secrets + # Run the shared test script source ./script/include/run_test