Scripts for finding accidental secrets in the repo.
This adds the following: - A detect-secrets dependency and a related script (`script/detect_secrets`) to find and alert developers to secrets added to the code. By default, the script will search staged and new, unstaged files. It can optionally search only staged files. - A whitelist, `.secrets.baseline`, that tracks instances of secrets or false positives already in the repo. - Modifies `script/test` to detect secrets as part of the test suite. - Updates to the README regarding the use of detect-secrets.
This commit is contained in:
dandds
parent
a941cca5e6
commit
2a0168b1e6
187
.secrets.baseline
Normal file
187
.secrets.baseline
Normal file
@ -0,0 +1,187 @@
|
||||
{
|
||||
"exclude": {
|
||||
"files": null,
|
||||
"lines": null
|
||||
},
|
||||
"generated_at": "2019-08-13T09:59:21Z",
|
||||
"plugins_used": [
|
||||
{
|
||||
"base64_limit": 4.5,
|
||||
"name": "Base64HighEntropyString"
|
||||
},
|
||||
{
|
||||
"name": "BasicAuthDetector"
|
||||
},
|
||||
{
|
||||
"hex_limit": 3,
|
||||
"name": "HexHighEntropyString"
|
||||
},
|
||||
{
|
||||
"name": "KeywordDetector"
|
||||
},
|
||||
{
|
||||
"name": "PrivateKeyDetector"
|
||||
}
|
||||
],
|
||||
"results": {
|
||||
"Pipfile.lock": [
|
||||
{
|
||||
"hashed_secret": "a355e3e3983231194cfc5981d48ea7ff9a7f9cb6",
|
||||
"is_secret": false,
|
||||
"is_verified": false,
|
||||
"line_number": 4,
|
||||
"type": "Hex High Entropy String"
|
||||
}
|
||||
],
|
||||
"README.md": [
|
||||
{
|
||||
"hashed_secret": "d141ce86b0584abb29ee7c24af9afb1e3d871f04",
|
||||
"is_secret": false,
|
||||
"is_verified": false,
|
||||
"line_number": 145,
|
||||
"type": "Secret Keyword"
|
||||
}
|
||||
],
|
||||
"alembic.ini": [
|
||||
{
|
||||
"hashed_secret": "9d4e1e23bd5b727046a9e3b4b7db57bd8d6ee684",
|
||||
"is_secret": false,
|
||||
"is_verified": false,
|
||||
"line_number": 38,
|
||||
"type": "Basic Auth Credentials"
|
||||
}
|
||||
],
|
||||
"alembic/versions/e0c6eb21771f_reset_migrations_with_new_schema.py": [
|
||||
{
|
||||
"hashed_secret": "999a22300a564f9d2bdca555c2170465fd760ae3",
|
||||
"is_secret": false,
|
||||
"is_verified": false,
|
||||
"line_number": 13,
|
||||
"type": "Hex High Entropy String"
|
||||
}
|
||||
],
|
||||
"atst.ini.example": [
|
||||
{
|
||||
"hashed_secret": "abcdb568713c255c81376829da20004ba9463fd3",
|
||||
"is_secret": false,
|
||||
"is_verified": false,
|
||||
"line_number": 2,
|
||||
"type": "Secret Keyword"
|
||||
}
|
||||
],
|
||||
"config/base.ini": [
|
||||
{
|
||||
"hashed_secret": "aa419433d95be86df254d499243bee1d5173f1ae",
|
||||
"is_secret": false,
|
||||
"is_verified": false,
|
||||
"line_number": 5,
|
||||
"type": "Secret Keyword"
|
||||
},
|
||||
{
|
||||
"hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3",
|
||||
"is_secret": false,
|
||||
"is_verified": false,
|
||||
"line_number": 17,
|
||||
"type": "Secret Keyword"
|
||||
},
|
||||
{
|
||||
"hashed_secret": "abcdb568713c255c81376829da20004ba9463fd3",
|
||||
"is_secret": false,
|
||||
"is_verified": false,
|
||||
"line_number": 23,
|
||||
"type": "Secret Keyword"
|
||||
}
|
||||
],
|
||||
"ssl/certificate-authority/ca.key": [
|
||||
{
|
||||
"hashed_secret": "be4fc4886bd949b369d5e092eb87494f12e57e5b",
|
||||
"is_secret": false,
|
||||
"is_verified": false,
|
||||
"line_number": 1,
|
||||
"type": "Private Key"
|
||||
}
|
||||
],
|
||||
"ssl/client-certs/atat.mil.key": [
|
||||
{
|
||||
"hashed_secret": "be4fc4886bd949b369d5e092eb87494f12e57e5b",
|
||||
"is_secret": false,
|
||||
"is_verified": false,
|
||||
"line_number": 1,
|
||||
"type": "Private Key"
|
||||
}
|
||||
],
|
||||
"ssl/client-certs/bad-atat.mil.key": [
|
||||
{
|
||||
"hashed_secret": "be4fc4886bd949b369d5e092eb87494f12e57e5b",
|
||||
"is_secret": false,
|
||||
"is_verified": false,
|
||||
"line_number": 1,
|
||||
"type": "Private Key"
|
||||
}
|
||||
],
|
||||
"ssl/client-certs/client-ca.key": [
|
||||
{
|
||||
"hashed_secret": "be4fc4886bd949b369d5e092eb87494f12e57e5b",
|
||||
"is_secret": false,
|
||||
"is_verified": false,
|
||||
"line_number": 1,
|
||||
"type": "Private Key"
|
||||
}
|
||||
],
|
||||
"ssl/server-certs/dev.cac.atat.codes.key": [
|
||||
{
|
||||
"hashed_secret": "be4fc4886bd949b369d5e092eb87494f12e57e5b",
|
||||
"is_secret": false,
|
||||
"is_verified": false,
|
||||
"line_number": 1,
|
||||
"type": "Private Key"
|
||||
}
|
||||
],
|
||||
"tests/acceptance/conftest.py": [
|
||||
{
|
||||
"hashed_secret": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f",
|
||||
"is_secret": false,
|
||||
"is_verified": false,
|
||||
"line_number": 48,
|
||||
"type": "Basic Auth Credentials"
|
||||
}
|
||||
],
|
||||
"tests/fixtures/chain/make-chain.sh": [
|
||||
{
|
||||
"hashed_secret": "bad2e396920ce37fe53fc291f90b130d915375fb",
|
||||
"is_secret": false,
|
||||
"is_verified": false,
|
||||
"line_number": 35,
|
||||
"type": "Secret Keyword"
|
||||
}
|
||||
],
|
||||
"tests/forms/test_validators.py": [
|
||||
{
|
||||
"hashed_secret": "260408f687da9094705a841acda9b029563053ee",
|
||||
"is_secret": false,
|
||||
"is_verified": false,
|
||||
"line_number": 30,
|
||||
"type": "Hex High Entropy String"
|
||||
}
|
||||
],
|
||||
"tests/routes/task_orders/test_new.py": [
|
||||
{
|
||||
"hashed_secret": "e4f14805dfd1e6af030359090c535e149e6b4207",
|
||||
"is_secret": false,
|
||||
"is_verified": false,
|
||||
"line_number": 30,
|
||||
"type": "Hex High Entropy String"
|
||||
}
|
||||
],
|
||||
"tests/test_access.py": [
|
||||
{
|
||||
"hashed_secret": "e4f14805dfd1e6af030359090c535e149e6b4207",
|
||||
"is_secret": false,
|
||||
"is_verified": false,
|
||||
"line_number": 481,
|
||||
"type": "Hex High Entropy String"
|
||||
}
|
||||
]
|
||||
},
|
||||
"version": "0.12.5"
|
||||
}
|
||||
1
Pipfile
1
Pipfile
@ -43,6 +43,7 @@ selenium = "*"
|
||||
honcho = "*"
|
||||
blinker = "*"
|
||||
pytest-mock = "*"
|
||||
detect-secrets = "*"
|
||||
|
||||
[requires]
|
||||
python_version = "3.7.3"
|
||||
|
||||
51
Pipfile.lock
generated
51
Pipfile.lock
generated
@ -1,7 +1,7 @@
|
||||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "d697cd7c279a761283ea2ffefab08d5854e4c0341b5646839b269ab0d999bf87"
|
||||
"sha256": "f09d90a1b4b86eff2e0ed453ceefcce4f19d54c57fb50ebed1f02aae8ab83c2a"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
@ -71,18 +71,18 @@
|
||||
},
|
||||
"boto3": {
|
||||
"hashes": [
|
||||
"sha256:666f37c5852f71925494fc2103b189deafe6702c1d9ae60bead5b1b6466de857",
|
||||
"sha256:7b77b507221ec15550b02d492804166bcc61ef3a81312968065515a76aa1791b"
|
||||
"sha256:3ec5b520dbd0a430cdd581a8250991fb0f21ee7e668a8928f15006b312fa86dc",
|
||||
"sha256:8aec0247131a0db1e33d28ad13910e01e6dfa208e8ab8ee5a4095e92dbaabf45"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.9.202"
|
||||
"version": "==1.9.204"
|
||||
},
|
||||
"botocore": {
|
||||
"hashes": [
|
||||
"sha256:71ca578701e746fe947c098e5dee06128d0f6ba98217ba7e29aff0dab8caf82f",
|
||||
"sha256:e55003c46e71396a551d4b70f39286f8fc4094ac6cf90f5db8d7a68bb4af1f9d"
|
||||
"sha256:56cd1114e0ce35733e890b321160c8c438243f4fa54d3d074dfa6bdce4ee55aa",
|
||||
"sha256:f86504bcc9c44d5b2e7b019f2f279b70f17b1400d2fc4775bc009ec473530cad"
|
||||
],
|
||||
"version": "==1.12.202"
|
||||
"version": "==1.12.204"
|
||||
},
|
||||
"certifi": {
|
||||
"hashes": [
|
||||
@ -562,6 +562,20 @@
|
||||
"index": "pypi",
|
||||
"version": "==1.4"
|
||||
},
|
||||
"certifi": {
|
||||
"hashes": [
|
||||
"sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939",
|
||||
"sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695"
|
||||
],
|
||||
"version": "==2019.6.16"
|
||||
},
|
||||
"chardet": {
|
||||
"hashes": [
|
||||
"sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
|
||||
"sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
|
||||
],
|
||||
"version": "==3.0.4"
|
||||
},
|
||||
"click": {
|
||||
"hashes": [
|
||||
"sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13",
|
||||
@ -615,6 +629,14 @@
|
||||
],
|
||||
"version": "==4.4.0"
|
||||
},
|
||||
"detect-secrets": {
|
||||
"hashes": [
|
||||
"sha256:33593cbf7099ced595e7371fcee31c815a237419de2869d571bbdb651a125d4c",
|
||||
"sha256:7b8e6d521f5a164ff17dbe3d2691eb85f842133d4e6bb7a23eeb461a0ab4e215"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.12.5"
|
||||
},
|
||||
"docopt": {
|
||||
"hashes": [
|
||||
"sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"
|
||||
@ -666,6 +688,13 @@
|
||||
"index": "pypi",
|
||||
"version": "==1.0.1"
|
||||
},
|
||||
"idna": {
|
||||
"hashes": [
|
||||
"sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
|
||||
"sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
|
||||
],
|
||||
"version": "==2.8"
|
||||
},
|
||||
"importlib-metadata": {
|
||||
"hashes": [
|
||||
"sha256:23d3d873e008a513952355379d93cbcab874c58f4f034ff657c7a87422fa64e8",
|
||||
@ -945,6 +974,14 @@
|
||||
"index": "pypi",
|
||||
"version": "==5.1.2"
|
||||
},
|
||||
"requests": {
|
||||
"hashes": [
|
||||
"sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
|
||||
"sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2.22.0"
|
||||
},
|
||||
"selenium": {
|
||||
"hashes": [
|
||||
"sha256:7491b5391f29a74774d475456d3a138e00fae0b3966f68a100f1f3ad331ce166",
|
||||
|
||||
35
README.md
35
README.md
@ -256,3 +256,38 @@ location /login-dev {
|
||||
```
|
||||
|
||||
The location block will require the same proxy pass configuration as other location blocks for the app.
|
||||
|
||||
## Secrets Detection
|
||||
|
||||
This project uses [detect-secrets](https://github.com/Yelp/detect-secrets) to help prevent secrets from being checked into source control. Secret detection is run automatically as part of `script/test` and can be run separately with `script/detect_secrets`.
|
||||
|
||||
If you need to check in a file that raises false positives from `detect-secrets`, you can add it to the whitelist. Run:
|
||||
|
||||
```
|
||||
pipenv run detect-secrets scan --update .secrets.baseline
|
||||
```
|
||||
|
||||
and then:
|
||||
|
||||
```
|
||||
pipenv run detect-secrets audit .secrets.baseline
|
||||
```
|
||||
|
||||
The audit will open an interactive prompt where you can whitelist the file. This is useful if you're checking in an entire file that looks like or is a secret (like a sample PKI file).
|
||||
|
||||
Alternatively, you can add a `# pragma: allowlist secret` comment to the line that raised the false positive. See the [detect-secret](https://github.com/Yelp/detect-secrets#inline-allowlisting) docs for more information.
|
||||
|
||||
It's recommended that you add a pre-commit hook to invoke `script/detect_secrets`. Add the example below or something equivalent to `.git/hooks/pre-commit`:
|
||||
|
||||
```
|
||||
#!/usr/bin/env bash
|
||||
|
||||
if ./script/detect_secrets staged; then
|
||||
echo "secrets check passed"
|
||||
else
|
||||
echo -e "**SECRETS DETECTED**"
|
||||
exit 1
|
||||
fi
|
||||
```
|
||||
|
||||
Also note that if the line number of a previously whitelisted secret changes, the whitelist file, `.secrets.baseline`, will be updated and needs to be committed.
|
||||
|
||||
43
script/detect_secrets
Executable file
43
script/detect_secrets
Executable file
@ -0,0 +1,43 @@
|
||||
#! .venv/bin/python
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from detect_secrets.pre_commit_hook import main as find_secrets
|
||||
|
||||
|
||||
TRACKED_CHANGES = ["git", "diff", "HEAD", "--name-only"]
|
||||
STAGED_CHANGES = ["git", "diff", "--cached", "--name-only"]
|
||||
UNTRACKED_CHANGES = ["git", "ls-files", "--others", "--exclude-standard"]
|
||||
|
||||
|
||||
def git_file_list(cmd):
|
||||
comproc = subprocess.run(cmd, capture_output=True)
|
||||
return [f.decode() for f in comproc.stdout.split()]
|
||||
|
||||
|
||||
def git_staged_files():
|
||||
return git_file_list(STAGED_CHANGES)
|
||||
|
||||
|
||||
def git_all_files():
|
||||
return git_file_list(TRACKED_CHANGES) + git_file_list(UNTRACKED_CHANGES)
|
||||
|
||||
|
||||
def main(arg):
|
||||
"""
|
||||
If `arg` is "staged", this will only check files that have been
|
||||
staged to the git index. Otherwise, it will check staged and
|
||||
unstaged files.
|
||||
"""
|
||||
files = []
|
||||
if arg == "staged":
|
||||
files = git_staged_files()
|
||||
else:
|
||||
files = git_all_files()
|
||||
|
||||
return find_secrets(["--baseline", ".secrets.baseline"] + files)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
arg = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
sys.exit(main(arg))
|
||||
@ -24,5 +24,8 @@ RUN_JS_TESTS="true"
|
||||
# Check python formatting
|
||||
source ./script/format check
|
||||
|
||||
# Check for secrets
|
||||
./script/detect_secrets
|
||||
|
||||
# Run the shared test script
|
||||
source ./script/include/run_test
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user