From 82f23761fdc1568462fe0587a48e13618f2b4059 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20M=C3=BCller?= Date: Sat, 7 Mar 2020 20:12:27 +0100 Subject: [PATCH 1/4] JSON Output working --- args.go | 11 ++++- main.go | 2 +- response.go | 137 +++++++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 137 insertions(+), 13 deletions(-) diff --git a/args.go b/args.go index 90cca52..821de03 100644 --- a/args.go +++ b/args.go @@ -50,6 +50,7 @@ type config struct { saveStatus saveStatusArgs timeout int verbose bool + jsonOut bool paths string hosts string @@ -106,6 +107,11 @@ func processArgs() config { flag.BoolVar(&rawHTTP, "rawhttp", false, "") flag.BoolVar(&rawHTTP, "r", false, "") + // Json Output param + jsonOut := false + flag.BoolVar(&jsonOut, "jsonOut", false, "") + flag.BoolVar(&jsonOut, "j", false, "") + // no headers noHeaders := false flag.BoolVar(&noHeaders, "no-headers", false, "") @@ -156,6 +162,7 @@ func processArgs() config { hosts: hosts, output: output, noHeaders: noHeaders, + jsonOut: jsonOut, } } @@ -176,7 +183,9 @@ func init() { h += " -s, --savestatus Save only responses with specific status code\n" h += " -t, --timeout Set the HTTP timeout (default: 10000)\n" h += " -v, --verbose Verbose mode\n" - h += " -X, --method HTTP method (default: GET)\n\n" + h += " -X, --method HTTP method (default: GET)\n" + h += " --no-header Don't output headers\n" + h += " -j, --jsonOut Output in JSON\n\n" h += "Defaults:\n" h += " pathsFile: ./paths\n" diff --git a/main.go b/main.go index f2a73a2..f1d9e3f 100644 --- a/main.go +++ b/main.go @@ -93,7 +93,7 @@ func main() { continue } - path, err := res.save(c.output, c.noHeaders) + path, err := res.save(c.output, c.noHeaders, c.jsonOut) if err != nil { fmt.Fprintf(os.Stderr, "failed to save file: %s\n", err) } diff --git a/response.go b/response.go index 1ca0d0a..88d691e 100644 --- a/response.go +++ b/response.go @@ -7,6 +7,8 @@ import ( "io/ioutil" "os" "path" + "encoding/json" + "strings" ) // a response is a wrapper around an HTTP response; @@ -22,51 +24,164 @@ type response struct { } // String returns a string representation of the request and response -func (r response) String() string { +func (r response) jsonString(noHeaders bool) string { + if noHeaders{ + // Falsch TODO + b := &bytes.Buffer{} + b.Write(r.body) + return b.String() + } + b := &bytes.Buffer{} b.WriteString(r.request.URL()) b.WriteString("\n\n") - b.WriteString(fmt.Sprintf("> %s %s HTTP/1.1\n", r.request.method, r.request.path)) + b.WriteString(fmt.Sprintf("%s %s HTTP/1.1\n", r.request.method, r.request.path)) // request headers for _, h := range r.request.headers { - b.WriteString(fmt.Sprintf("> %s\n", h)) + b.WriteString(fmt.Sprintf("%s\n", h)) } b.WriteString("\n") // status line - b.WriteString(fmt.Sprintf("< HTTP/1.1 %s\n", r.status)) + b.WriteString(fmt.Sprintf("HTTP/1.1 %s\n", r.status)) // response headers for _, h := range r.headers { - b.WriteString(fmt.Sprintf("< %s\n", h)) + b.WriteString(fmt.Sprintf("%s\n", h)) } b.WriteString("\n") // body - b.Write(r.body) - + /* + r.headers + r.status + r.err + */ + type JsonHeader struct{ + JsonHeaderKey string `json:"headerkey"` + JsonHeaderValue string `json:"headervalue"` + } + type JsonRequest struct { + JsonRequestUrl string `json:"url"` + JsonRequestHostname string `json:"hostname"` + JsonRequestMethod string `json:"method"` + JsonRequestPath string `json:"path"` + JsonRequestHost string `json:"host"` + JsonRequestHeaders []JsonHeader `json:"headers"` + JsonRequestBody string `json:"body"` + JsonRequestFollow bool `json:"follow"` + } + r.request.Hostname() + type JsonResponse struct { + JsonResponseStatusCode int `json:"statuscode"` + JsonResponseStatus string `json:"status"` + JsonResponseHeader []JsonHeader `json:"headers"` + JsonBody string `json:"body"` + } + type JsonOut struct { + JsonRequest JsonRequest `json:"request"` + JsonResponse JsonResponse `json:"response"` + } + var jsonRequestHeaders []JsonHeader + for _, h := range r.request.headers { + x := strings.Split(h, ":") + jsonRequestHeaders = append(jsonRequestHeaders, JsonHeader{x[0], strings.TrimSpace(x[1])}) + } + var jsonResponseHeaders []JsonHeader + for _, h := range r.headers { + x := strings.Split(h, ":") + jsonResponseHeaders = append(jsonResponseHeaders, JsonHeader{x[0], strings.TrimSpace(x[1])}) + } + //r.h + // var jsonRequestHeader []JsonHeader + // for _, h := range r.request.headers { + // x := strings.Split(h, ":") + // jsonRequestHeader = append(jsonRequestHeader, JsonHeader{x[0], strings.TrimSpace(x[1])}) + // } + // //r.headers + x := &bytes.Buffer{} + x.Write(r.body) + x.String() + resp := JsonOut{ + JsonRequest{ + r.request.URL(), + r.request.Hostname(), + r.request.method, + r.request.path, + r.request.host, + jsonRequestHeaders, + r.request.body, + r.request.followLocation, + }, + JsonResponse{ + r.statusCode, + r.status, + jsonResponseHeaders, + x.String(), + }, + } + ba, err := json.Marshal(resp) + if err != nil { + fmt.Println("error:", err) + } + fmt.Printf("%s", ba) return b.String() } -func (r response) StringNoHeaders() string { +// String returns a string representation of the request and response +func (r response) megString(noHeaders bool) string { + if noHeaders{ + b := &bytes.Buffer{} + b.Write(r.body) + return b.String() + } + b := &bytes.Buffer{} + b.WriteString(r.request.URL()) + b.WriteString("\n\n") + + b.WriteString(fmt.Sprintf("> %s %s HTTP/1.1\n", r.request.method, r.request.path)) + + // request headers + for _, h := range r.request.headers { + b.WriteString(fmt.Sprintf("> %s\n", h)) + } + b.WriteString("\n") + + // status line + b.WriteString(fmt.Sprintf("< HTTP/1.1 %s\n", r.status)) + + // response headers + for _, h := range r.headers { + b.WriteString(fmt.Sprintf("< %s\n", h)) + } + b.WriteString("\n") + + // body b.Write(r.body) return b.String() } + // save write a request and response output to disk -func (r response) save(pathPrefix string, noHeaders bool) (string, error) { +func (r response) save(pathPrefix string, noHeaders bool, json bool) (string, error) { + var content []byte - content := []byte(r.String()) + if json{ + content = []byte(r.jsonString(noHeaders)) + }else{ + content = []byte(r.megString(noHeaders)) + } +/* if noHeaders { content = []byte(r.StringNoHeaders()) } - +*/ checksum := sha1.Sum(content) parts := []string{pathPrefix} From 649af622deb7b7c80161f60ad8ab9d621715077c Mon Sep 17 00:00:00 2001 From: Timo Mueller Date: Tue, 17 Mar 2020 21:55:12 +0100 Subject: [PATCH 2/4] Code cleanup and testing some functionality --- response.go | 153 ++++++++++++++++++++++------------------------------ 1 file changed, 63 insertions(+), 90 deletions(-) diff --git a/response.go b/response.go index 88d691e..08273db 100644 --- a/response.go +++ b/response.go @@ -3,11 +3,11 @@ package main import ( "bytes" "crypto/sha1" + "encoding/json" "fmt" "io/ioutil" "os" "path" - "encoding/json" "strings" ) @@ -23,89 +23,64 @@ type response struct { err error } -// String returns a string representation of the request and response +// jsonString returns a JSON string representation of the request and response func (r response) jsonString(noHeaders bool) string { - if noHeaders{ - // Falsch TODO - b := &bytes.Buffer{} - b.Write(r.body) - return b.String() - } - - b := &bytes.Buffer{} - - b.WriteString(r.request.URL()) - b.WriteString("\n\n") + // Building the structs for the various JSON elements (root element, headers, body, ...) - b.WriteString(fmt.Sprintf("%s %s HTTP/1.1\n", r.request.method, r.request.path)) - - // request headers - for _, h := range r.request.headers { - b.WriteString(fmt.Sprintf("%s\n", h)) - } - b.WriteString("\n") - - // status line - b.WriteString(fmt.Sprintf("HTTP/1.1 %s\n", r.status)) - - // response headers - for _, h := range r.headers { - b.WriteString(fmt.Sprintf("%s\n", h)) - } - b.WriteString("\n") - - // body - /* - r.headers - r.status - r.err - */ - type JsonHeader struct{ - JsonHeaderKey string `json:"headerkey"` + // Struct for Headers (Key, Value) + type JsonHeader struct { + JsonHeaderKey string `json:"headerkey"` JsonHeaderValue string `json:"headervalue"` } + // Struct representing the request (URL, Hostname, HTTP Method, Path, Host, Headers (struct), Body, Follow (-L)) type JsonRequest struct { - JsonRequestUrl string `json:"url"` - JsonRequestHostname string `json:"hostname"` - JsonRequestMethod string `json:"method"` - JsonRequestPath string `json:"path"` - JsonRequestHost string `json:"host"` - JsonRequestHeaders []JsonHeader `json:"headers"` - JsonRequestBody string `json:"body"` - JsonRequestFollow bool `json:"follow"` - } - r.request.Hostname() + JsonRequestUrl string `json:"url"` + JsonRequestHostname string `json:"hostname"` + JsonRequestMethod string `json:"method"` + JsonRequestPath string `json:"path"` + JsonRequestHost string `json:"host"` + JsonRequestHeaders []JsonHeader `json:"headers"` + JsonRequestBody string `json:"body"` + JsonRequestFollow bool `json:"follow"` + } + // Struct representing the response (HTTP status code, status text, headers(struct), body) type JsonResponse struct { - JsonResponseStatusCode int `json:"statuscode"` - JsonResponseStatus string `json:"status"` - JsonResponseHeader []JsonHeader `json:"headers"` - JsonBody string `json:"body"` + JsonResponseStatusCode int `json:"statuscode"` + JsonResponseStatus string `json:"status"` + JsonResponseHeader []JsonHeader `json:"headers"` + JsonBody string `json:"body"` } + // Struct representing the root element (Request, Response) type JsonOut struct { - JsonRequest JsonRequest `json:"request"` + JsonRequest JsonRequest `json:"request"` JsonResponse JsonResponse `json:"response"` } + + // Prepare the Header structs which will be inserted into the JSON element var jsonRequestHeaders []JsonHeader - for _, h := range r.request.headers { - x := strings.Split(h, ":") - jsonRequestHeaders = append(jsonRequestHeaders, JsonHeader{x[0], strings.TrimSpace(x[1])}) - } var jsonResponseHeaders []JsonHeader - for _, h := range r.headers { - x := strings.Split(h, ":") - jsonResponseHeaders = append(jsonResponseHeaders, JsonHeader{x[0], strings.TrimSpace(x[1])}) + // If we specify --no-headers we will simply receive "headers":null in both the request and response + // Otherwise we will extract the headers from the request/response and fill the header elements + if !noHeaders { + // Fill the request header struct + for _, h := range r.request.headers { + x := strings.Split(h, ":") + jsonRequestHeaders = append(jsonRequestHeaders, JsonHeader{x[0], strings.TrimSpace(x[1])}) + } + // Fill the response header struct + for _, h := range r.headers { + x := strings.Split(h, ":") + jsonResponseHeaders = append(jsonResponseHeaders, JsonHeader{x[0], strings.TrimSpace(x[1])}) + } } - //r.h - // var jsonRequestHeader []JsonHeader - // for _, h := range r.request.headers { - // x := strings.Split(h, ":") - // jsonRequestHeader = append(jsonRequestHeader, JsonHeader{x[0], strings.TrimSpace(x[1])}) - // } - // //r.headers - x := &bytes.Buffer{} - x.Write(r.body) - x.String() + // Create the JSON Body element + jsonBody := &bytes.Buffer{} + jsonBody.Write(r.body) + + // Create the JSON element + // Root Element resp := JsonOut{ + // Request Element JsonRequest{ r.request.URL(), r.request.Hostname(), @@ -115,25 +90,27 @@ func (r response) jsonString(noHeaders bool) string { jsonRequestHeaders, r.request.body, r.request.followLocation, - }, - JsonResponse{ - r.statusCode, - r.status, - jsonResponseHeaders, - x.String(), - }, - } + }, + // Response Element + JsonResponse{ + r.statusCode, + r.status, + jsonResponseHeaders, + jsonBody.String(), + }, + } + // Marshal the JSON struct ba, err := json.Marshal(resp) if err != nil { fmt.Println("error:", err) } - fmt.Printf("%s", ba) - return b.String() + + return string(ba) } -// String returns a string representation of the request and response +// megString returns a string representation of the request and response in a "traditional" meg format func (r response) megString(noHeaders bool) string { - if noHeaders{ + if noHeaders { b := &bytes.Buffer{} b.Write(r.body) return b.String() @@ -167,21 +144,17 @@ func (r response) megString(noHeaders bool) string { return b.String() } - // save write a request and response output to disk func (r response) save(pathPrefix string, noHeaders bool, json bool) (string, error) { var content []byte - if json{ + // Depending if we want to save a json or meg string we call the corresponding String method + if json { content = []byte(r.jsonString(noHeaders)) - }else{ + } else { content = []byte(r.megString(noHeaders)) } -/* - if noHeaders { - content = []byte(r.StringNoHeaders()) - } -*/ + checksum := sha1.Sum(content) parts := []string{pathPrefix} From 83420b22832e69680a9b7d4d640f1bcee1b9c689 Mon Sep 17 00:00:00 2001 From: Timo Mueller Date: Tue, 17 Mar 2020 22:04:18 +0100 Subject: [PATCH 3/4] Fixed error with Headers containing more then one colon --- response.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/response.go b/response.go index 08273db..1590168 100644 --- a/response.go +++ b/response.go @@ -64,12 +64,13 @@ func (r response) jsonString(noHeaders bool) string { if !noHeaders { // Fill the request header struct for _, h := range r.request.headers { - x := strings.Split(h, ":") + // We use SplitN because then we only split on the first colon occurrence + x := strings.SplitN(h, ":", 2) jsonRequestHeaders = append(jsonRequestHeaders, JsonHeader{x[0], strings.TrimSpace(x[1])}) } // Fill the response header struct for _, h := range r.headers { - x := strings.Split(h, ":") + x := strings.SplitN(h, ":", 2) jsonResponseHeaders = append(jsonResponseHeaders, JsonHeader{x[0], strings.TrimSpace(x[1])}) } } From 3f1d9fb5619012e8b2e5cf20ca04900626c09ee7 Mon Sep 17 00:00:00 2001 From: Timo Mueller Date: Tue, 17 Mar 2020 22:24:10 +0100 Subject: [PATCH 4/4] Updated docs/help for the json output. --- README.mkd | 14 ++++++++++++++ args.go | 6 +++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/README.mkd b/README.mkd index 0a8bf16..60fe190 100644 --- a/README.mkd +++ b/README.mkd @@ -244,3 +244,17 @@ out/example.com/61ac5fbb9d3dd054006ae82630b045ba730d8618:3:> TRACE /robots.txt H out/example.com/bd8d9f4c470ffa0e6ec8cfa8ba1c51d62289b6dd:3:> TRACE /.well-known/security.txt HTTP/1.1 ... ``` + +### JSON Output + +Instead of the usual meg output it is also possible to output the results +as a JSON string with the `-j` or `--jsonOut` option: + +``` +▶ meg --jsonOut +{"request":{"url":"","hostname":"","method":"","path":"","host":"","headers":[{"headerkey":"","headervalue":""},{"headerkey":"","headervalue":""}],"body":"","follow":false},"response":{"statuscode":,"status":"","headers":[{"headerkey":"","headervalue":""}],"body":""}} +``` + +### No headers + +With --no-headers you suppress the output of HTTP headers. diff --git a/args.go b/args.go index 821de03..694e0e0 100644 --- a/args.go +++ b/args.go @@ -50,7 +50,7 @@ type config struct { saveStatus saveStatusArgs timeout int verbose bool - jsonOut bool + jsonOut bool paths string hosts string @@ -162,7 +162,7 @@ func processArgs() config { hosts: hosts, output: output, noHeaders: noHeaders, - jsonOut: jsonOut, + jsonOut: jsonOut, } } @@ -184,7 +184,7 @@ func init() { h += " -t, --timeout Set the HTTP timeout (default: 10000)\n" h += " -v, --verbose Verbose mode\n" h += " -X, --method HTTP method (default: GET)\n" - h += " --no-header Don't output headers\n" + h += " --no-headers Don't output headers\n" h += " -j, --jsonOut Output in JSON\n\n" h += "Defaults:\n"