Mercurial > gemma
changeset 409:cdd63547930a
Rename external wfs proxy to proxy as it may work for other kind os services like wms, too.
author | Sascha L. Teichmann <sascha.teichmann@intevation.de> |
---|---|
date | Wed, 15 Aug 2018 15:57:36 +0200 |
parents | ac23905e64b1 |
children | 3f803d64a6ee |
files | controllers/externalwfs.go controllers/proxy.go |
diffstat | 2 files changed, 380 insertions(+), 380 deletions(-) [+] |
line wrap: on
line diff
--- a/controllers/externalwfs.go Wed Aug 15 15:55:41 2018 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,380 +0,0 @@ -package controllers - -import ( - "compress/flate" - "compress/gzip" - "crypto/hmac" - "crypto/sha256" - "encoding/base64" - "encoding/xml" - "io" - "io/ioutil" - "log" - "net/http" - "net/url" - "regexp" - "strings" - "time" - - "gemma.intevation.de/gemma/config" - "github.com/gorilla/mux" - "golang.org/x/net/html/charset" -) - -// proxyBlackList is a set of URLs that should not be rewritten by the proxy. -var proxyBlackList = map[string]struct{}{ - "http://www.w3.org/2001/XMLSchema-instance": struct{}{}, - "http://www.w3.org/1999/xlink": struct{}{}, - "http://www.w3.org/2001/XMLSchema": struct{}{}, - "http://www.w3.org/XML/1998/namespace": struct{}{}, - "http://www.opengis.net/wfs/2.0": struct{}{}, - "http://www.opengis.net/ows/1.1": struct{}{}, - "http://www.opengis.net/gml/3.2": struct{}{}, - "http://www.opengis.net/fes/2.0": struct{}{}, - "http://schemas.opengis.net/gml": struct{}{}, -} - -func findEntry(entry string) (string, bool) { - external := config.ExternalWFSs() - if external == nil || len(external) == 0 { - return "", false - } - alias, found := external[entry] - if !found { - return "", false - } - data, ok := alias.(map[string]interface{}) - if !ok { - return "", false - } - urlS, found := data["url"] - if !found { - return "", false - } - url, ok := urlS.(string) - return url, ok -} - -func proxyDirector(req *http.Request) { - - log.Printf("proxyDirector: %s\n", req.RequestURI) - - abort := func(format string, args ...interface{}) { - log.Printf(format, args...) - panic(http.ErrAbortHandler) - } - - vars := mux.Vars(req) - - var s string - - if entry, found := vars["entry"]; found { - if s, found = findEntry(entry); !found { - abort("Cannot find entry '%s'\n", entry) - } - } else { - expectedMAC, err := base64.URLEncoding.DecodeString(vars["hash"]) - if err != nil { - abort("Cannot base64 decode hash: %v\n", err) - } - url, err := base64.URLEncoding.DecodeString(vars["url"]) - if err != nil { - abort("Cannot base64 decode url: %v\n", err) - } - - mac := hmac.New(sha256.New, config.ProxyKey()) - mac.Write(url) - messageMAC := mac.Sum(nil) - - s = string(url) - - if !hmac.Equal(messageMAC, expectedMAC) { - abort("HMAC of URL %s failed.\n", s) - } - } - - nURL := s + "?" + req.URL.RawQuery - //log.Printf("%v\n", nURL) - - u, err := url.Parse(nURL) - if err != nil { - abort("Invalid url: %v\n", err) - } - req.URL = u - - req.Host = u.Host - //req.Header.Del("If-None-Match") - //log.Printf("headers: %v\n", req.Header) -} - -type nopCloser struct { - io.Writer -} - -func (nopCloser) Close() error { return nil } - -func encoding(h http.Header) ( - func(io.Reader) (io.ReadCloser, error), - func(io.Writer) (io.WriteCloser, error), -) { - switch enc := h.Get("Content-Encoding"); { - case strings.Contains(enc, "gzip"): - log.Println("gzip compression") - return func(r io.Reader) (io.ReadCloser, error) { - return gzip.NewReader(r) - }, - func(w io.Writer) (io.WriteCloser, error) { - return gzip.NewWriter(w), nil - } - case strings.Contains(enc, "deflate"): - log.Println("Deflate compression") - return func(r io.Reader) (io.ReadCloser, error) { - return flate.NewReader(r), nil - }, - func(w io.Writer) (io.WriteCloser, error) { - return flate.NewWriter(w, flate.DefaultCompression) - } - default: - log.Println("No content compression") - return func(r io.Reader) (io.ReadCloser, error) { - if r2, ok := r.(io.ReadCloser); ok { - return r2, nil - } - return ioutil.NopCloser(r), nil - }, - func(w io.Writer) (io.WriteCloser, error) { - if w2, ok := w.(io.WriteCloser); ok { - return w2, nil - } - return nopCloser{w}, nil - } - } -} - -func proxyModifyResponse(resp *http.Response) error { - - if !isXML(resp.Header) { - return nil - } - - pr, pw := io.Pipe() - - var ( - r io.ReadCloser - w io.WriteCloser - err error - ) - - reader, writer := encoding(resp.Header) - - if r, err = reader(resp.Body); err != nil { - return err - } - - if w, err = writer(pw); err != nil { - return err - } - - go func(force io.ReadCloser) { - start := time.Now() - defer func() { - //r.Close() - w.Close() - pw.Close() - force.Close() - log.Printf("rewrite took %s\n", time.Since(start)) - }() - if err := rewrite(w, r); err != nil { - log.Printf("rewrite failed: %v\n", err) - return - } - log.Println("rewrite successful") - }(resp.Body) - - resp.Body = pr - - return nil -} - -var xmlContentTypes = []string{ - "application/xml", - "text/xml", - "application/gml+xml", -} - -func isXML(h http.Header) bool { - for _, t := range h["Content-Type"] { - t = strings.ToLower(t) - for _, ct := range xmlContentTypes { - if strings.Contains(t, ct) { - return true - } - } - } - return false -} - -var replaceRe = regexp.MustCompile(`\b(https?://[^\s\?]*)`) - -func replace(s string) string { - - proxyKey := config.ProxyKey() - proxyPrefix := config.ProxyPrefix() + "/api/proxy/" - - return replaceRe.ReplaceAllStringFunc(s, func(s string) string { - if _, found := proxyBlackList[s]; found { - return s - } - mac := hmac.New(sha256.New, proxyKey) - b := []byte(s) - mac.Write(b) - expectedMAC := mac.Sum(nil) - - hash := base64.URLEncoding.EncodeToString(expectedMAC) - enc := base64.URLEncoding.EncodeToString(b) - return proxyPrefix + hash + "/" + enc - }) -} - -func rewrite(w io.Writer, r io.Reader) error { - - decoder := xml.NewDecoder(r) - decoder.CharsetReader = charset.NewReaderLabel - - encoder := xml.NewEncoder(w) - - var n nsdef - -tokens: - for { - tok, err := decoder.Token() - switch { - case tok == nil && err == io.EOF: - break tokens - case err != nil: - return err - } - - switch t := tok.(type) { - case xml.StartElement: - t = t.Copy() - - isDef := n.isDef(t.Name.Space) - n = n.push() - - for i := range t.Attr { - t.Attr[i].Value = replace(t.Attr[i].Value) - n.checkDef(&t.Attr[i]) - } - - for i := range t.Attr { - n.adjust(&t.Attr[i]) - } - - switch { - case isDef: - t.Name.Space = "" - default: - if s := n.lookup(t.Name.Space); s != "" { - t.Name.Space = "" - t.Name.Local = s + ":" + t.Name.Local - } - } - tok = t - - case xml.CharData: - tok = xml.CharData(replace(string(t))) - - case xml.EndElement: - s := n.lookup(t.Name.Space) - - n = n.pop() - - if n.isDef(t.Name.Space) { - t.Name.Space = "" - } else if s != "" { - t.Name.Space = "" - t.Name.Local = s + ":" + t.Name.Local - } - tok = t - } - - if err := encoder.EncodeToken(tok); err != nil { - return err - } - } - - return encoder.Flush() -} - -type nsframe struct { - def string - ns map[string]string -} - -type nsdef []nsframe - -func (n nsdef) setDef(def string) { - if l := len(n); l > 0 { - n[l-1].def = def - } -} - -func (n nsdef) isDef(s string) bool { - for i := len(n) - 1; i >= 0; i-- { - if x := n[i].def; x != "" { - return s == x - } - } - return false -} - -func (n nsdef) define(ns, s string) { - if l := len(n); l > 0 { - n[l-1].ns[ns] = s - } -} - -func (n nsdef) lookup(ns string) string { - for i := len(n) - 1; i >= 0; i-- { - if s := n[i].ns[ns]; s != "" { - return s - } - } - return "" -} - -func (n nsdef) checkDef(at *xml.Attr) { - if at.Name.Space == "" && at.Name.Local == "xmlns" { - n.setDef(at.Value) - } -} - -func (n nsdef) adjust(at *xml.Attr) { - switch { - case at.Name.Space == "xmlns": - n.define(at.Value, at.Name.Local) - at.Name.Local = "xmlns:" + at.Name.Local - at.Name.Space = "" - - case at.Name.Space != "": - if n.isDef(at.Name.Space) { - at.Name.Space = "" - } else if s := n.lookup(at.Name.Space); s != "" { - at.Name.Local = s + ":" + at.Name.Local - at.Name.Space = "" - } - } -} - -func (n nsdef) push() nsdef { - return append(n, nsframe{ns: make(map[string]string)}) -} - -func (n nsdef) pop() nsdef { - if l := len(n); l > 0 { - n[l-1] = nsframe{} - n = n[:l-1] - } - return n -}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/controllers/proxy.go Wed Aug 15 15:57:36 2018 +0200 @@ -0,0 +1,380 @@ +package controllers + +import ( + "compress/flate" + "compress/gzip" + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/xml" + "io" + "io/ioutil" + "log" + "net/http" + "net/url" + "regexp" + "strings" + "time" + + "gemma.intevation.de/gemma/config" + "github.com/gorilla/mux" + "golang.org/x/net/html/charset" +) + +// proxyBlackList is a set of URLs that should not be rewritten by the proxy. +var proxyBlackList = map[string]struct{}{ + "http://www.w3.org/2001/XMLSchema-instance": struct{}{}, + "http://www.w3.org/1999/xlink": struct{}{}, + "http://www.w3.org/2001/XMLSchema": struct{}{}, + "http://www.w3.org/XML/1998/namespace": struct{}{}, + "http://www.opengis.net/wfs/2.0": struct{}{}, + "http://www.opengis.net/ows/1.1": struct{}{}, + "http://www.opengis.net/gml/3.2": struct{}{}, + "http://www.opengis.net/fes/2.0": struct{}{}, + "http://schemas.opengis.net/gml": struct{}{}, +} + +func findEntry(entry string) (string, bool) { + external := config.ExternalWFSs() + if external == nil || len(external) == 0 { + return "", false + } + alias, found := external[entry] + if !found { + return "", false + } + data, ok := alias.(map[string]interface{}) + if !ok { + return "", false + } + urlS, found := data["url"] + if !found { + return "", false + } + url, ok := urlS.(string) + return url, ok +} + +func proxyDirector(req *http.Request) { + + log.Printf("proxyDirector: %s\n", req.RequestURI) + + abort := func(format string, args ...interface{}) { + log.Printf(format, args...) + panic(http.ErrAbortHandler) + } + + vars := mux.Vars(req) + + var s string + + if entry, found := vars["entry"]; found { + if s, found = findEntry(entry); !found { + abort("Cannot find entry '%s'\n", entry) + } + } else { + expectedMAC, err := base64.URLEncoding.DecodeString(vars["hash"]) + if err != nil { + abort("Cannot base64 decode hash: %v\n", err) + } + url, err := base64.URLEncoding.DecodeString(vars["url"]) + if err != nil { + abort("Cannot base64 decode url: %v\n", err) + } + + mac := hmac.New(sha256.New, config.ProxyKey()) + mac.Write(url) + messageMAC := mac.Sum(nil) + + s = string(url) + + if !hmac.Equal(messageMAC, expectedMAC) { + abort("HMAC of URL %s failed.\n", s) + } + } + + nURL := s + "?" + req.URL.RawQuery + //log.Printf("%v\n", nURL) + + u, err := url.Parse(nURL) + if err != nil { + abort("Invalid url: %v\n", err) + } + req.URL = u + + req.Host = u.Host + //req.Header.Del("If-None-Match") + //log.Printf("headers: %v\n", req.Header) +} + +type nopCloser struct { + io.Writer +} + +func (nopCloser) Close() error { return nil } + +func encoding(h http.Header) ( + func(io.Reader) (io.ReadCloser, error), + func(io.Writer) (io.WriteCloser, error), +) { + switch enc := h.Get("Content-Encoding"); { + case strings.Contains(enc, "gzip"): + log.Println("gzip compression") + return func(r io.Reader) (io.ReadCloser, error) { + return gzip.NewReader(r) + }, + func(w io.Writer) (io.WriteCloser, error) { + return gzip.NewWriter(w), nil + } + case strings.Contains(enc, "deflate"): + log.Println("Deflate compression") + return func(r io.Reader) (io.ReadCloser, error) { + return flate.NewReader(r), nil + }, + func(w io.Writer) (io.WriteCloser, error) { + return flate.NewWriter(w, flate.DefaultCompression) + } + default: + log.Println("No content compression") + return func(r io.Reader) (io.ReadCloser, error) { + if r2, ok := r.(io.ReadCloser); ok { + return r2, nil + } + return ioutil.NopCloser(r), nil + }, + func(w io.Writer) (io.WriteCloser, error) { + if w2, ok := w.(io.WriteCloser); ok { + return w2, nil + } + return nopCloser{w}, nil + } + } +} + +func proxyModifyResponse(resp *http.Response) error { + + if !isXML(resp.Header) { + return nil + } + + pr, pw := io.Pipe() + + var ( + r io.ReadCloser + w io.WriteCloser + err error + ) + + reader, writer := encoding(resp.Header) + + if r, err = reader(resp.Body); err != nil { + return err + } + + if w, err = writer(pw); err != nil { + return err + } + + go func(force io.ReadCloser) { + start := time.Now() + defer func() { + //r.Close() + w.Close() + pw.Close() + force.Close() + log.Printf("rewrite took %s\n", time.Since(start)) + }() + if err := rewrite(w, r); err != nil { + log.Printf("rewrite failed: %v\n", err) + return + } + log.Println("rewrite successful") + }(resp.Body) + + resp.Body = pr + + return nil +} + +var xmlContentTypes = []string{ + "application/xml", + "text/xml", + "application/gml+xml", +} + +func isXML(h http.Header) bool { + for _, t := range h["Content-Type"] { + t = strings.ToLower(t) + for _, ct := range xmlContentTypes { + if strings.Contains(t, ct) { + return true + } + } + } + return false +} + +var replaceRe = regexp.MustCompile(`\b(https?://[^\s\?]*)`) + +func replace(s string) string { + + proxyKey := config.ProxyKey() + proxyPrefix := config.ProxyPrefix() + "/api/proxy/" + + return replaceRe.ReplaceAllStringFunc(s, func(s string) string { + if _, found := proxyBlackList[s]; found { + return s + } + mac := hmac.New(sha256.New, proxyKey) + b := []byte(s) + mac.Write(b) + expectedMAC := mac.Sum(nil) + + hash := base64.URLEncoding.EncodeToString(expectedMAC) + enc := base64.URLEncoding.EncodeToString(b) + return proxyPrefix + hash + "/" + enc + }) +} + +func rewrite(w io.Writer, r io.Reader) error { + + decoder := xml.NewDecoder(r) + decoder.CharsetReader = charset.NewReaderLabel + + encoder := xml.NewEncoder(w) + + var n nsdef + +tokens: + for { + tok, err := decoder.Token() + switch { + case tok == nil && err == io.EOF: + break tokens + case err != nil: + return err + } + + switch t := tok.(type) { + case xml.StartElement: + t = t.Copy() + + isDef := n.isDef(t.Name.Space) + n = n.push() + + for i := range t.Attr { + t.Attr[i].Value = replace(t.Attr[i].Value) + n.checkDef(&t.Attr[i]) + } + + for i := range t.Attr { + n.adjust(&t.Attr[i]) + } + + switch { + case isDef: + t.Name.Space = "" + default: + if s := n.lookup(t.Name.Space); s != "" { + t.Name.Space = "" + t.Name.Local = s + ":" + t.Name.Local + } + } + tok = t + + case xml.CharData: + tok = xml.CharData(replace(string(t))) + + case xml.EndElement: + s := n.lookup(t.Name.Space) + + n = n.pop() + + if n.isDef(t.Name.Space) { + t.Name.Space = "" + } else if s != "" { + t.Name.Space = "" + t.Name.Local = s + ":" + t.Name.Local + } + tok = t + } + + if err := encoder.EncodeToken(tok); err != nil { + return err + } + } + + return encoder.Flush() +} + +type nsframe struct { + def string + ns map[string]string +} + +type nsdef []nsframe + +func (n nsdef) setDef(def string) { + if l := len(n); l > 0 { + n[l-1].def = def + } +} + +func (n nsdef) isDef(s string) bool { + for i := len(n) - 1; i >= 0; i-- { + if x := n[i].def; x != "" { + return s == x + } + } + return false +} + +func (n nsdef) define(ns, s string) { + if l := len(n); l > 0 { + n[l-1].ns[ns] = s + } +} + +func (n nsdef) lookup(ns string) string { + for i := len(n) - 1; i >= 0; i-- { + if s := n[i].ns[ns]; s != "" { + return s + } + } + return "" +} + +func (n nsdef) checkDef(at *xml.Attr) { + if at.Name.Space == "" && at.Name.Local == "xmlns" { + n.setDef(at.Value) + } +} + +func (n nsdef) adjust(at *xml.Attr) { + switch { + case at.Name.Space == "xmlns": + n.define(at.Value, at.Name.Local) + at.Name.Local = "xmlns:" + at.Name.Local + at.Name.Space = "" + + case at.Name.Space != "": + if n.isDef(at.Name.Space) { + at.Name.Space = "" + } else if s := n.lookup(at.Name.Space); s != "" { + at.Name.Local = s + ":" + at.Name.Local + at.Name.Space = "" + } + } +} + +func (n nsdef) push() nsdef { + return append(n, nsframe{ns: make(map[string]string)}) +} + +func (n nsdef) pop() nsdef { + if l := len(n); l > 0 { + n[l-1] = nsframe{} + n = n[:l-1] + } + return n +}