diff --git a/.gitignore b/.gitignore index 7f015a2..5f00a3b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ /build /od-ash-tray /ash-tray +.env +/cdn/* +test* diff --git a/Dockerfile b/Dockerfile index 2d475ae..281bf8b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,9 +6,4 @@ RUN CGO_ENABLED=0 GOOS=linux go build -o /od-ash-tray . FROM scratch COPY --from=build /od-ash-tray /od-ash-tray -# bandaid fix -# we'll have to fix this later... -COPY ./fullchain.pem fullchain.pem -COPY ./privkey.pem privkey.pem -COPY ./ash.yaml ./ash.yaml CMD ["/od-ash-tray"] diff --git a/Jenkinsfile b/Jenkinsfile index 9324602..3924e2d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,13 +8,6 @@ pipeline { } stages { - stage('Certs Copy (ISSUE)') { - steps { - sh 'cp /etc/letsencrypt/archive/onlinedi.vision/fullchain2.pem fullchain.pem' - sh 'cp /etc/letsencrypt/archive/onlinedi.vision/privkey2.pem privkey.pem' - sh 'touch ash.yaml' - } - } stage('DEL_CDN Setup') { steps { sh 'stat ~/del_cdn 2> /dev/null > /dev/null || mkdir ~/del_cdn' @@ -39,11 +32,8 @@ pipeline { stage('First Shadow') { steps{ sh 'file_sufix=$(curl -i -X POST -H "Content-Type: multipart/form-data" -F "data=@shadows/1.png" -k https://127.0.0.1:7377/upload | tail -n1); \ - echo "${file_sufix}"; \ curl "https://127.0.0.1:7377/${file_sufix}" -k -o test1 ;\ - count=$(wc -l shadows/1.png | cut -f1 -d" "); \ - diffr=$(sdiff -B -b -s test1 shadows/1.png | wc -l); \ - exit $(( $diffr*100/$count )); \ + diff test1 shadows/1.png \ ' } } @@ -51,9 +41,7 @@ pipeline { steps{ sh 'file_sufix=$(curl -i -X POST -H "Content-Type: multipart/form-data" -F "data=@shadows/2.png" -k https://127.0.0.1:7377/upload | tail -n1); \ curl -k "https://127.0.0.1:7377/${file_sufix}" -o test2; \ - count=$(wc -l shadows/2.png | cut -f1 -d" "); \ - diffr=$(sdiff -B -b -s test2 shadows/2.png | wc -l); \ - exit $(( $diffr*100/$count )); \ + diff test2 shadows/2.png \ ' } } @@ -61,9 +49,7 @@ pipeline { steps { sh 'file_sufix=$(curl -i -X POST -H "Content-Type: multipart/form-data" -F "data=@shadows/3.png" -k https://127.0.0.1:7377/upload | tail -n1); \ curl -k "https://127.0.0.1:7377/${file_sufix}" -o test3; \ - count=$(wc -l shadows/3.png | cut -f1 -d" "); \ - diffr=$(sdiff -B -b -s test3 shadows/3.png | wc -l); \ - exit $(( $diffr*100/$count )); \ + diff test3 shadows/3.png \ ' } } @@ -72,9 +58,7 @@ pipeline { steps { sh 'file_sufix=$(curl -i -X POST -H "Content-Type: multipart/form-data" -F "data=@shadows/4.png" -k https://127.0.0.1:7377/upload | tail -n1); \ curl -k "https://127.0.0.1:7377/${file_sufix}" -o test4; \ - count=$(wc -l shadows/4.png | cut -f1 -d" "); \ - diffr=$(sdiff -B -b -s test4 shadows/4.png | wc -l); \ - exit $(( $diffr*100/$count )); \ + diff test4 shadows/4.png \ ' } } @@ -83,20 +67,18 @@ pipeline { steps { sh 'file_sufix=$(curl -i -X POST -H "Content-Type: multipart/form-data" -F "data=@shadows/5.png" -k https://127.0.0.1:7377/upload | tail -n1); \ curl -k "https://127.0.0.1:7377/${file_sufix}" -o test5; \ - count=$(wc -l shadows/5.png | cut -f1 -d" "); \ - diffr=$(sdiff -B -b -s test5 shadows/5.png | wc -l); \ - exit $(( $diffr*100/$count )); \ + diff test5 shadows/5.png \ ' } } } } - // stage('Cleanup Shadow Arena') { - // steps { - // sh 'rm -rf ~/del_cdn' - // } - // } + stage('Cleanup Shadow Arena') { + steps { + sh 'rm -rf ~/del_cdn' + } + } stage('Docker Kill') { steps { @@ -116,10 +98,5 @@ pipeline { } } } - stage('Certs Cleanup') { - steps { - sh 'rm privkey.pem fullchain.pem' - } - } } } diff --git a/ci-Dockerfile b/ci-Dockerfile index 22d9b62..0004169 100644 --- a/ci-Dockerfile +++ b/ci-Dockerfile @@ -3,9 +3,4 @@ WORKDIR /app COPY . . RUN CGO_ENABLED=0 GOOS=linux go build -o /od-ash-tray . -FROM scratch -COPY --from=build /od-ash-tray /od-ash-tray - -# bandaid fix -# we'll have to fix this later... CMD ["/od-ash-tray"] diff --git a/ci-compose.yaml b/ci-compose.yaml index 6b9e70b..e1535b9 100644 --- a/ci-compose.yaml +++ b/ci-compose.yaml @@ -5,11 +5,9 @@ services: ports: - "0.0.0.0:7377:7377" environment: - - ASH_TRAY_ID + - ASH_TRAY_ID=ash - ASH_TRAY_DIRECTORY=/cdn - - ASH_TRAY_KEY + - ASH_TRAY_KEY=aaaaaVVVVVVVAAAAAAAAaaaaaaaaaaaa - ASH_TRAY_PORT=7377 volumes: - - /cdn:/cdn:rw - - /etc/letsencrypt/live/onlinedi.vision:/certs:r - + - ./cdn:/cdn:rw diff --git a/compose.yaml b/compose.yaml index a81ea02..066d83b 100644 --- a/compose.yaml +++ b/compose.yaml @@ -2,7 +2,7 @@ services: od-ash_tray: build: . ports: - - "0.0.0.0:7377:7377" + - "127.0.0.1:7377:7377" environment: - ASH_TRAY_ID - ASH_TRAY_DIRECTORY=/cdn @@ -10,5 +10,3 @@ services: - ASH_TRAY_PORT=7377 volumes: - /cdn:/cdn:rw - - /etc/letsencrypt/live/onlinedi.vision:/certs:r - diff --git a/launch-test-env.sh b/launch-test-env.sh new file mode 100755 index 0000000..dd63795 --- /dev/null +++ b/launch-test-env.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +set -eu + +COMPOSE_FLAGS=(-f ci-compose.yaml) +ASH_HOST="http://127.0.0.1:7377" + +function print_usage() { + echo "Usage: ./launch-test-env.sh [OPTION]" + echo "OPTION:" + echo " -c clean the environment (stop processes / kill docker) " + echo " -W complete wipeout of the environment" + echo " -t run tests..." + echo " -b skip rebuild" +} + + +function cleanup() { + echo "========= CLEANING UP COMPOSE ===========" + docker compose "${COMPOSE_FLAGS[@]}" down +} + +function wipeout() { + echo "========= COMPLETE WIPEOUT OF ENV ===========" + docker compose "${COMPOSE_FLAGS[@]}" down --rmi 'all' + sudo rm -rf cdn + rm -rf test* +} + +trap 'cleanup' SIGINT SIGTERM + +t_flag='' +b_flag='' +while getopts 'htcWb' flag; do + case "${flag}" in + b) b_flag='true';; + t) t_flag='true';; + c) cleanup + exit 0;; + W) wipeout + exit 0;; + h) print_usage + exit 0 ;; + *) print_usage + exit 1 ;; + esac +done + +echo "========= BUILDING PROJECT ===========" +if ! [[ "${b_flag}" == "true" ]]; then + docker compose "${COMPOSE_FLAGS[@]}" build --no-cache +fi +docker compose "${COMPOSE_FLAGS[@]}" up -d + +if [[ "$t_flag" == "true" ]]; then + echo "========= RUNNING TESTS ===========" + for file in ./shadows/* ; do + printf " TESTING: %s" "${file}..." + cdn_file=$(curl -ksi -X POST -H "Content-Type: multipart/form-data" -F "data=@${file}" "${ASH_HOST}"/ash/upload | tail -n1) + curl -ks "${ASH_HOST}"/"${cdn_file}" -o test + diff test "${file}" + echo " PASSED" + done + + cleanup +fi + diff --git a/layout.go b/layout.go deleted file mode 100644 index e45d01f..0000000 --- a/layout.go +++ /dev/null @@ -1,29 +0,0 @@ -package main - -import ( - "log" - "os" - "gopkg.in/yaml.v2" -) - - -type Ash struct { - Server string - Address string - AshID string -} - -func newAshes() []Ash { - content, err := os.ReadFile("ash.yaml") - if err != nil { - log.Fatal(err) - } - - var ashes []Ash - - if err := yaml.Unmarshal(content, &ashes); err != nil { - log.Fatal(err) - } - - return ashes -} diff --git a/main.go b/main.go index c60e7a5..29ab884 100644 --- a/main.go +++ b/main.go @@ -3,32 +3,29 @@ package main import ( "crypto/aes" "crypto/sha256" - "crypto/tls" + "crypto/cipher" + "crypto/rand" + "encoding/binary" "encoding/hex" "fmt" - "github.com/google/uuid" "io" "log" "net/http" "os" "strings" "time" + + "github.com/google/uuid" ) -// tigger jenkins var ( - ashID = os.Getenv("ASH_TRAY_ID") + ashID = "ash" ashTrayDir = os.Getenv("ASH_TRAY_DIRECTORY") ashHighSignature = os.Getenv("ASH_HIGH_SIGNATURE") ashLowSignature = os.Getenv("ASH_LOW_SIGNATURE") ashKey = []byte(os.Getenv("ASH_TRAY_KEY")) ) -var ( - CertFilePath = "/fullchain.pem" - KeyFilePath = "/privkey.pem" -) - const ( MaxFormMemorySize = 20000000 encSize = 16096 @@ -53,9 +50,25 @@ func createNewAshTray(req *http.Request) (string, string) { return "", "" } + // TODO: see if we could use something stronger then + // SHA256 here... maybe argon2? sha := sha256.Sum256([]byte(dirUUID[:])) directory := hex.EncodeToString(sha[:]) filename := hex.EncodeToString(newUUID[:]) + + // TODO: BUG: CRITICAL: VULNERABILITY: URGENT: FIX: + // + // Here instead of using 'filename' we should use a hashed 'filename'. + // Then we should return the UN-hashed filename alongside the filePath + // and directory variables. The UN-hashed filename variable should be + // combined with the ashKey and be used for encryption/decryption. + // It should then be returned to the user. Via URL. + // + // We also need to change the way we give users their files. When a + // user requests a file (with the UN-hashed filename) we will hash + // that key and use it for decryption. + // + // Any reason to also hash the directory name? return fmt.Sprintf("%s/%s/%s.%s", ashID, directory, filename, extension), directory } @@ -73,38 +86,70 @@ func writeToFile(filePath string, directory string, data []byte) { } func encryptData(data string) ([]byte, error) { - + aesBlock, err := aes.NewCipher(ashKey) if err != nil { return nil, err } - encryptedData := make([]byte, encSize) - - bytesOfData := make([]byte, encSize) - copy(bytesOfData[:], data) - - for iter := range encSize / BlockSize { - offset := iter * BlockSize - aesBlock.Encrypt(encryptedData[offset:offset+BlockSize], bytesOfData[offset:offset+BlockSize]) + + gcm, err := cipher.NewGCM(aesBlock) + if err != nil { + return nil, err } - - return encryptedData, nil + + nonce := make([]byte, gcm.NonceSize()) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return nil, err + } + + ciphertext := gcm.Seal(nil, nonce, []byte(data), nil) + + encryptedLen := len(nonce) + len(ciphertext) + result := make([]byte, 4+encryptedLen) + binary.BigEndian.PutUint32(result[:4], uint32(encryptedLen)) + copy(result[4:], nonce) + copy(result[4+len(nonce):], ciphertext) + + return result, nil } func decryptData(data []byte) ([]byte, error) { - + + if len(data) < 4 { + return nil, fmt.Errorf("data too short") + } + + expectedLen := int(binary.BigEndian.Uint32(data[:4])) + if len(data) < 4+expectedLen { + return nil, fmt.Errorf("incomplete data") + } + + encryptedData := data[4 : 4+expectedLen] + aesBlock, err := aes.NewCipher(ashKey) if err != nil { return nil, err } - decryptedData := make([]byte, encSize) - - for iter := range encSize / BlockSize { - offset := iter * BlockSize - aesBlock.Decrypt(decryptedData[offset:offset+BlockSize], data[offset:offset+BlockSize]) + + gcm, err := cipher.NewGCM(aesBlock) + if err != nil { + return nil, err } - - return decryptedData, nil + + nonceSize := gcm.NonceSize() + if len(encryptedData) < nonceSize { + return nil, fmt.Errorf("encrypted data too short") + } + + nonce := encryptedData[:nonceSize] + ciphertext := encryptedData[nonceSize:] + + plaintext, err := gcm.Open(nil, nonce, ciphertext, nil) + if err != nil { + return nil, err + } + + return plaintext, nil } func fileDownload(httpWriter http.ResponseWriter, req *http.Request) { @@ -114,32 +159,63 @@ func fileDownload(httpWriter http.ResponseWriter, req *http.Request) { file, err := os.Open(filePath) if err != nil { fmt.Printf(" %s file doesn't exist\n", filePath) - fmt.Fprintf(httpWriter, "HTTP/1.1 404 Not Found \r\n") + + httpWriter.WriteHeader(404) + fmt.Fprintf(httpWriter, "404 Not Found \r\n") return } defer file.Close() - buffer := make([]byte, encSize) + buffer := make([]byte, 65536) + var leftover []byte fmt.Println(" sending file") + httpWriter.WriteHeader(200) + for { readTotal, err := file.Read(buffer) - if err != nil { - if err != io.EOF { + if err != nil && err != io.EOF { + httpWriter.WriteHeader(500) + fmt.Fprintf(httpWriter, "Failed at reading from file.") + break + } + + if readTotal == 0 { + break + } + + data := append(leftover, buffer[:readTotal]...) + leftover = nil + + offset := 0 + for offset+4 <= len(data) { + blockLen := int(binary.BigEndian.Uint32(data[offset:offset+4])) + if offset+4+blockLen > len(data) { + leftover = data[offset:] + break + } + + decryptedData, err := decryptData(data[offset : offset+4+blockLen]) + if err != nil { fmt.Println(err) + httpWriter.WriteHeader(500) + fmt.Fprintf(httpWriter, "Failed at file decryption.") + return } - break + + httpWriter.Write(decryptedData) + offset += 4 + blockLen + } + + if offset < len(data) && leftover == nil { + leftover = data[offset:] } - decryptedData, err := decryptData(buffer[:readTotal]) - if err != nil { + + if err == io.EOF { break } - httpWriter.WriteHeader(200) - fmt.Fprintf(httpWriter, "%s", string(decryptedData[:])) - } - } func fileUpload(httpWriter http.ResponseWriter, req *http.Request) { @@ -184,13 +260,12 @@ func higherTrayTimer() func() { } } -func ashGet(httpWriter http.ResponseWriter, req *http.Request, ashes []Ash) { +func ashGet(httpWriter http.ResponseWriter, req *http.Request) { fileDownload(httpWriter, req) } func higherTray(httpWriter http.ResponseWriter, req *http.Request) { defer higherTrayTimer()() - ashes := newAshes() fmt.Printf("[%s] %s: %s\n", req.Method, req.RemoteAddr, req.URL) req.ParseMultipartForm(MaxFormMemorySize) @@ -198,12 +273,12 @@ func higherTray(httpWriter http.ResponseWriter, req *http.Request) { httpWriter.Header().Set("Access-Control-Allow-Origin", "*") if req.Method == "GET" { - if req.URL.Path == "/ping" { + if req.URL.Path == "/ash/ping" { fmt.Fprintf(httpWriter, "ping") } else { - ashGet(httpWriter, req, ashes) + ashGet(httpWriter, req) } - } else if req.Method == "POST" && req.URL.Path == "/upload" { + } else if req.Method == "POST" && req.URL.Path == "/ash/upload" { fileUpload(httpWriter, req) } else { httpWriter.WriteHeader(400) @@ -214,7 +289,7 @@ func higherTray(httpWriter http.ResponseWriter, req *http.Request) { func main() { if ashTrayDir == "" { - fmt.Println("ASH_TRAY_DIR env var MUST be set") + fmt.Println("ASH_TRAY_DIRECTORY env var MUST be set") return } @@ -223,23 +298,12 @@ func main() { fmt.Println(err) } - serverTLSCert, err := tls.LoadX509KeyPair(CertFilePath, KeyFilePath) - if err != nil { - fmt.Println("COULD NOT LOAD TLS CERTIFICATE... BAILING OUT...") - return - } - - tlsConfig := &tls.Config{ - Certificates: []tls.Certificate{serverTLSCert}, - } - tray_server := &http.Server{ - Addr: fmt.Sprintf(":%s", os.Getenv("ASH_TRAY_PORT")), + Addr: fmt.Sprintf("0.0.0.0:%s", os.Getenv("ASH_TRAY_PORT")), Handler: http.HandlerFunc(higherTray), ReadTimeout: 10 * time.Second, WriteTimeout: 10 * time.Second, MaxHeaderBytes: 1 << 20, - TLSConfig: tlsConfig, } - log.Fatal(tray_server.ListenAndServeTLS("", "")) + log.Fatal(tray_server.ListenAndServe()) }