# HG changeset patch # User Sascha L. Teichmann # Date 1548839000 -3600 # Node ID 079a1d35e076bd4e079ef3c38995392c1249aa0c # Parent b4ba751e70a11de9f9ecd79d63b3757965a70bf5# Parent 51a7917b6a0843a077516f1270f3f4af8ff63ed2 Merged 'unify_imports' branch back into default. diff -r b4ba751e70a1 -r 079a1d35e076 client/src/components/Sidebar.vue --- a/client/src/components/Sidebar.vue Tue Jan 29 12:36:45 2019 +0100 +++ b/client/src/components/Sidebar.vue Wed Jan 30 10:03:20 2019 +0100 @@ -138,7 +138,7 @@
- + { - this.$store.commit("imports/setImportScheduleDetailVisible"); + this.$store.commit("importschedule/setImportScheduleDetailVisible"); }) .catch(error => { const { status, data } = error.response; @@ -170,7 +170,7 @@ }); }, getSchedules() { - this.$store.dispatch("imports/loadSchedules").catch(error => { + this.$store.dispatch("importschedule/loadSchedules").catch(error => { const { status, data } = error.response; displayError({ title: this.$gettext("Backend Error"), @@ -179,12 +179,12 @@ }); }, newImport() { - this.$store.commit("imports/setImportScheduleDetailVisible"); + this.$store.commit("importschedule/setImportScheduleDetailVisible"); }, deleteSchedule(index) { if (this.importScheduleDetailVisible) return; this.$store - .dispatch("imports/deleteSchedule", index) + .dispatch("importschedule/deleteSchedule", index) .then(() => { this.getSchedules(); displayInfo({ @@ -203,7 +203,7 @@ }, computed: { ...mapState("application", ["showSidebar"]), - ...mapState("imports", ["schedules", "importScheduleDetailVisible"]), + ...mapState("importschedule", ["schedules", "importScheduleDetailVisible"]), activeStyle() { const color = this.importScheduleDetailVisible ? "#aeaeae" : "#000000"; return { color: color }; diff -r b4ba751e70a1 -r 079a1d35e076 client/src/components/importschedule/Importscheduledetail.vue --- a/client/src/components/importschedule/Importscheduledetail.vue Tue Jan 29 12:36:45 2019 +0100 +++ b/client/src/components/importschedule/Importscheduledetail.vue Wed Jan 30 10:03:20 2019 +0100 @@ -32,19 +32,19 @@ >Gauge measurement Available fairway fpths Waterway area Fairway dimension Distance marks virtual @@ -165,7 +165,7 @@
- Simple Schedule + Simple schedule
@@ -382,7 +382,7 @@ IMPORTTYPES, IMPORTTYPEKIND, initializeCurrentSchedule -} from "@/store/imports.js"; +} from "@/store/importschedule"; import { mapState } from "vuex"; import { displayInfo, displayError } from "@/lib/errors.js"; import app from "@/main.js"; @@ -413,44 +413,6 @@ ...initializeCurrentSchedule() }; }, - IMPORTTYPES: IMPORTTYPES, - EVERY: app.$gettext("Every"), - MINUTESPAST: app.$gettext("minutes past"), - ON: app.$gettext("on"), - OF: app.$gettext("of"), - AT: app.$gettext("at"), - OCLOCK: app.$gettext("o' clock"), - CRONMODE: { - "15minutes": app.$gettext("15 minutes"), - hour: app.$gettext("hour"), - day: app.$gettext("day"), - week: app.$gettext("week"), - month: app.$gettext("month"), - year: app.$gettext("year") - }, - DAYSOFWEEK: { - 1: app.$gettext("Monday"), - 2: app.$gettext("Tuesday"), - 3: app.$gettext("Wednesday"), - 4: app.$gettext("Thursday"), - 5: app.$gettext("Friday"), - 6: app.$gettext("Saturday"), - 0: app.$gettext("Sunday") - }, - MONTHS: { - 1: app.$gettext("January"), - 2: app.$gettext("February"), - 3: app.$gettext("March"), - 4: app.$gettext("April"), - 5: app.$gettext("May"), - 6: app.$gettext("June"), - 7: app.$gettext("July"), - 8: app.$gettext("August"), - 9: app.$gettext("September"), - 10: app.$gettext("October"), - 11: app.$gettext("November"), - 12: app.$gettext("December") - }, mounted() { this.initialize(); }, @@ -486,7 +448,10 @@ } }, computed: { - ...mapState("imports", ["importScheduleDetailVisible", "currentSchedule"]), + ...mapState("importschedule", [ + "importScheduleDetailVisible", + "currentSchedule" + ]), dialogLabel() { if (this.id) return this.$gettext("Import") + " " + this.id; return this.$gettext("New Import"); @@ -665,7 +630,10 @@ } this.triggerActive = false; this.$store - .dispatch("imports/triggerImport", { type: this.import_, data }) + .dispatch("importschedule/triggerImport", { + type: IMPORTTYPEKIND[this.import_], + data + }) .then(response => { const { id } = response.data; displayInfo({ @@ -685,10 +653,6 @@ }); }, save() { - const addAttribute = (data, attribute) => { - if (!data["attributes"]) data.attributes = {}; - data["attributes"] = { ...data["attributes"], ...attribute }; - }; if (!this.import_) return; let cron = this.cronString; if (this.easyCron) { @@ -696,31 +660,29 @@ if (this.simple === "monthly") cron = "0 0 0 1 * *"; } let data = {}; + let config = {}; + data["kind"] = IMPORTTYPEKIND[this.import_]; + if (this.isURLRequired) { if (!this.url) return; - data["url"] = this.url; - addAttribute(data, { - insecure: this.insecure + "" - }); + config["url"] = this.url; + config["insecure"] = this.insecure; } if (this.isSortbyRequired) { if (!this.sortBy) return; - addAttribute(data, { - "sort-by": this.sortBy - }); + config["sort-by"] = this.sortBy; } if (this.isFeatureTypeRequired) { if (!this.featureType) return; - addAttribute(data, { - "feature-type": this.featureType - }); + config["feature-type"] = this.featureType; } if (this.isCredentialsRequired) { if (!this.username || !this.password) return; - addAttribute(data, { + config = { + ...config, username: this.username, password: this.password - }); + }; } if (this.import_ == this.$options.IMPORTTYPES.FAIRWAYDIMENSION) { if ( @@ -731,21 +693,17 @@ !this.sourceOrganization ) return; - const values = { - los: this.LOS + "", - depth: this.depth + "" - }; - values["min-width"] = this.minWidth + ""; - values["max-width"] = this.maxWidth + ""; - values["source-organization"] = this.sourceOrganization; - addAttribute(data, values); + config = { ...config, los: this.LOS, depth: this.depth }; + config["min-width"] = this.minWidth; + config["max-width"] = this.maxWidth; + config["source-organization"] = this.sourceOrganization; } - if (this.scheduled) data["cron"] = cron; - data["kind"] = IMPORTTYPEKIND[this.import_]; - data["send-email"] = this.eMailNotification; + if (this.scheduled) config["cron"] = cron; + config["send-email"] = this.eMailNotification; + data["config"] = config; if (!this.id) { this.$store - .dispatch("imports/saveCurrentSchedule", data) + .dispatch("importschedule/saveCurrentSchedule", data) .then(response => { const { id } = response.data; displayInfo({ @@ -753,13 +711,15 @@ message: this.$gettext("Saved import: #") + id }); this.closeDetailview(); - this.$store.dispatch("imports/loadSchedules").catch(error => { - const { status, data } = error.response; - displayError({ - title: this.$gettext("Backend Error"), - message: `${status}: ${data.message || data}` + this.$store + .dispatch("importschedule/loadSchedules") + .catch(error => { + const { status, data } = error.response; + displayError({ + title: this.gettext("Backend Error"), + message: `${status}: ${data.message || data}` + }); }); - }); }) .catch(error => { const { status, data } = error.response; @@ -770,7 +730,7 @@ }); } else { this.$store - .dispatch("imports/updateCurrentSchedule", { + .dispatch("importschedule/updateCurrentSchedule", { data: data, id: this.id }) @@ -781,13 +741,15 @@ message: this.$gettext("update import: #") + id }); this.closeDetailview(); - this.$store.dispatch("imports/loadSchedules").catch(error => { - const { status, data } = error.response; - displayError({ - title: this.$gettext("Backend Error"), - message: `${status}: ${data.message || data}` + this.$store + .dispatch("importschedule/loadSchedules") + .catch(error => { + const { status, data } = error.response; + displayError({ + title: this.gettext("Backend Error"), + message: `${status}: ${data.message || data}` + }); }); - }); }) .catch(error => { const { status, data } = error.response; @@ -799,16 +761,49 @@ } }, closeDetailview() { - this.$store.commit("imports/clearCurrentSchedule"); - this.$store.commit("imports/setImportScheduleDetailInvisible"); + this.$store.commit("importschedule/clearCurrentSchedule"); + this.$store.commit("importschedule/setImportScheduleDetailInvisible"); } }, - imports: [], + IMPORTTYPES: IMPORTTYPES, on: "on", off: "off", - periods: { - DAILY: "daily", - MONTHLY: "monthly" + EVERY: app.$gettext("Every"), + MINUTESPAST: app.$gettext("minutes past"), + ON: app.$gettext("on"), + OF: app.$gettext("of"), + AT: app.$gettext("at"), + OCLOCK: app.$gettext("o' clock"), + CRONMODE: { + "15minutes": app.$gettext("15 minutes"), + hour: app.$gettext("hour"), + day: app.$gettext("day"), + week: app.$gettext("week"), + month: app.$gettext("month"), + year: app.$gettext("year") + }, + DAYSOFWEEK: { + 1: app.$gettext("Monday"), + 2: app.$gettext("Tuesday"), + 3: app.$gettext("Wednesday"), + 4: app.$gettext("Thursday"), + 5: app.$gettext("Friday"), + 6: app.$gettext("Saturday"), + 0: app.$gettext("Sunday") + }, + MONTHS: { + 1: app.$gettext("January"), + 2: app.$gettext("February"), + 3: app.$gettext("March"), + 4: app.$gettext("April"), + 5: app.$gettext("May"), + 6: app.$gettext("June"), + 7: app.$gettext("July"), + 8: app.$gettext("August"), + 9: app.$gettext("September"), + 10: app.$gettext("October"), + 11: app.$gettext("November"), + 12: app.$gettext("December") } }; diff -r b4ba751e70a1 -r 079a1d35e076 client/src/components/staging/StagingDetail.vue --- a/client/src/components/staging/StagingDetail.vue Tue Jan 29 12:36:45 2019 +0100 +++ b/client/src/components/staging/StagingDetail.vue Wed Jan 30 10:03:20 2019 +0100 @@ -13,7 +13,10 @@ data.summary.bottlenecks.length }}) - - + {{ data.summary["source-organization"] }} (LOS: + {{ data.summary.los }})
{{ data.kind.toUpperCase() }} diff -r b4ba751e70a1 -r 079a1d35e076 client/src/store/imports.js --- a/client/src/store/imports.js Tue Jan 29 12:36:45 2019 +0100 +++ b/client/src/store/imports.js Wed Jan 30 10:03:20 2019 +0100 @@ -13,7 +13,6 @@ */ import { HTTP } from "@/lib/http"; -import Vue from "vue"; import { WFS } from "ol/format.js"; import { equalTo as equalToFilter } from "ol/format/filter.js"; @@ -25,86 +24,12 @@ REJECTED: "declined" }; -const IMPORTTYPES = { - BOTTLENECK: "bottleneck", - WATERWAYAXIS: "waterwayaxis", - GAUGEMEASUREMENT: "gaugemeasurement", - FAIRWAYAVAILABILITY: "fairwayavailability", - WATERWAYAREA: "waterwayarea", - FAIRWAYDIMENSION: "fairwaydimension", - WATERWAYGAUGES: "waterwaygauges", - DISTANCEMARKSVIRTUAL: "distancemarksvirtual" -}; - -const SCHEDULES = { - DAILY: "daily", - MONTHLY: "monthly" -}; - -const IMPORTTYPEKIND = { - bottleneck: "bn", - fairwayavailability: "fa", - gaugemeasurement: "gm", - waterwayaxis: "wx", - waterwayarea: "wa", - fairwaydimension: "fd", - waterwaygauges: "wg", - distancemarksvirtual: "dmv" -}; - -const KINDIMPORTTYPE = { - bn: "bottleneck", - fa: "fairwayavailability", - gm: "gaugemeasurement", - wx: "waterwayaxis", - wa: "waterwayarea", - fd: "fairwaydimension", - wg: "waterwaygauge", - dmv: "distancemarksvirtual" -}; - -const initializeCurrentSchedule = () => { - return { - id: null, - importType: null, - schedule: null, - import_: null, - importSource: null, - eMailNotification: false, - scheduled: false, - easyCron: true, - cronString: "* * * * ", - cronMode: "", - minutes: null, - month: null, - hour: null, - day: null, - dayOfMonth: null, - simple: null, - url: null, - insecure: false, - triggerActive: true, - featureType: null, - sortBy: null, - username: "", - password: "", - LOS: 1, - minWidth: null, - maxWidth: null, - depth: null, - sourceOrganization: null - }; -}; - // initial state const init = () => { return { stretches: [], imports: [], staging: [], - schedules: [], - importScheduleDetailVisible: false, - currentSchedule: initializeCurrentSchedule(), importToReview: null }; }; @@ -117,18 +42,6 @@ setStretches: (state, stretches) => { state.stretches = stretches; }, - clearCurrentSchedule: state => { - state.currentSchedule = initializeCurrentSchedule(); - }, - setImportScheduleDetailInvisible: state => { - state.importScheduleDetailVisible = false; - }, - setImportScheduleDetailVisible: state => { - state.importScheduleDetailVisible = true; - }, - setSchedules: (state, schedules) => { - state.schedules = schedules; - }, setImports: (state, imports) => { state.imports = imports; }, @@ -153,66 +66,6 @@ } else { stagedResult.status = newStatus; } - }, - unmarshallCurrentSchedule: (state, payload) => { - const { kind, id, cron, url, attributes } = payload; - const eMailNotification = payload["send-email"]; - Vue.set(state.currentSchedule, "import_", KINDIMPORTTYPE[kind]); - Vue.set(state.currentSchedule, "id", id); - if (cron) { - Vue.set(state.currentSchedule, "scheduled", true); - Vue.set(state.currentSchedule, "easyCron", false); - Vue.set(state.currentSchedule, "cronString", cron); - } - if (eMailNotification) { - Vue.set(state.currentSchedule, "eMailNotification", eMailNotification); - } - if (url) { - Vue.set(state.currentSchedule, "url", url); - } - if (attributes) { - let { insecure, username, password, los, depth } = attributes; - let sortBy = attributes["sort-by"]; - let minWidth = attributes["min-width"]; - let maxWidth = attributes["max-width"]; - let sourceOrganization = attributes["source-organization"]; - const featureType = attributes["feature-type"]; - insecure = insecure == "true"; - if (insecure) { - Vue.set(state.currentSchedule, "insecure", insecure); - } - if (featureType) { - Vue.set(state.currentSchedule, "featureType", featureType); - } - if (sortBy) { - Vue.set(state.currentSchedule, "sortBy", sortBy); - } - if (username) { - Vue.set(state.currentSchedule, "username", username); - } - if (password) { - Vue.set(state.currentSchedule, "password", password); - } - if (los) { - Vue.set(state.currentSchedule, "LOS", los); - } - if (minWidth) { - Vue.set(state.currentSchedule, "minWidth", minWidth); - } - if (maxWidth) { - Vue.set(state.currentSchedule, "maxWidth", maxWidth); - } - if (depth) { - Vue.set(state.currentSchedule, "depth", depth); - } - if (sourceOrganization) { - Vue.set( - state.currentSchedule, - "sourceOrganization", - sourceOrganization - ); - } - } } }, actions: { @@ -264,95 +117,6 @@ }); }); }, - loadSchedule({ commit }, id) { - return new Promise((resolve, reject) => { - HTTP.get("/imports/config/" + id, { - headers: { "X-Gemma-Auth": localStorage.getItem("token") } - }) - .then(response => { - commit("unmarshallCurrentSchedule", response.data); - resolve(response); - }) - .catch(error => { - reject(error); - }); - }); - }, - deleteSchedule({ commit }, id) { - return new Promise((resolve, reject) => { - HTTP.delete("imports/config/" + id, { - headers: { - "X-Gemma-Auth": localStorage.getItem("token") - } - }) - .then(response => { - resolve(response); - }) - .catch(error => { - reject(error); - }); - }); - }, - updateCurrentSchedule({ commit }, payload) { - const { data, id } = payload; - return new Promise((resolve, reject) => { - HTTP.patch("imports/config/" + id, data, { - headers: { - "X-Gemma-Auth": localStorage.getItem("token") - } - }) - .then(response => { - resolve(response); - }) - .catch(error => { - reject(error); - }); - }); - }, - saveCurrentSchedule({ commit }, data) { - return new Promise((resolve, reject) => { - HTTP.post("imports/config", data, { - headers: { - "X-Gemma-Auth": localStorage.getItem("token") - } - }) - .then(response => { - resolve(response); - }) - .catch(error => { - reject(error); - }); - }); - }, - loadSchedules({ commit }) { - return new Promise((resolve, reject) => { - HTTP.get("/imports/config", { - headers: { "X-Gemma-Auth": localStorage.getItem("token") } - }) - .then(response => { - commit("setSchedules", response.data); - resolve(response); - }) - .catch(error => { - reject(error); - }); - }); - }, - triggerImport({ commit }, { type, data }) { - return new Promise((resolve, reject) => { - HTTP.post("imports/" + type, data, { - headers: { - "X-Gemma-Auth": localStorage.getItem("token") - } - }) - .then(response => { - resolve(response); - }) - .catch(error => { - reject(error); - }); - }); - }, getImports({ commit }) { return new Promise((resolve, reject) => { HTTP.get("/imports", { @@ -384,11 +148,4 @@ } }; -export { - imports, - STATES, - SCHEDULES, - IMPORTTYPES, - IMPORTTYPEKIND, - initializeCurrentSchedule -}; +export { imports, STATES }; diff -r b4ba751e70a1 -r 079a1d35e076 client/src/store/importschedule.js --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/client/src/store/importschedule.js Wed Jan 30 10:03:20 2019 +0100 @@ -0,0 +1,271 @@ +/* This is Free Software under GNU Affero General Public License v >= 3.0 + * without warranty, see README.md and license for details. + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * License-Filename: LICENSES/AGPL-3.0.txt + * + * Copyright (C) 2018 by via donau + * – Österreichische Wasserstraßen-Gesellschaft mbH + * Software engineering by Intevation GmbH + * + * Author(s): + * Thomas Junk + */ + +import Vue from "vue"; +import { HTTP } from "@/lib/http"; + +/* eslint-disable no-unused-vars */ +/* eslint-disable no-unreachable */ + +const IMPORTTYPES = { + BOTTLENECK: "bottleneck", + WATERWAYAXIS: "waterwayaxis", + GAUGEMEASUREMENT: "gaugemeasurement", + FAIRWAYAVAILABILITY: "fairwayavailability", + WATERWAYAREA: "waterwayarea", + FAIRWAYDIMENSION: "fairwaydimension", + WATERWAYGAUGES: "waterwaygauges", + DISTANCEMARKSVIRTUAL: "distancemarksvirtual" +}; + +const KINDIMPORTTYPE = { + bn: "bottleneck", + fa: "fairwayavailability", + gm: "gaugemeasurement", + wx: "waterwayaxis", + wa: "waterwayarea", + fd: "fairwaydimension", + wg: "waterwaygauge", + dmv: "distancemarksvirtual" +}; + +const IMPORTTYPEKIND = { + bottleneck: "bn", + fairwayavailability: "fa", + gaugemeasurement: "gm", + waterwayaxis: "wx", + waterwayarea: "wa", + fairwaydimension: "fd", + waterwaygauges: "wg", + distancemarksvirtual: "dmv" +}; + +const initializeCurrentSchedule = () => { + return { + id: null, + importType: null, + schedule: null, + import_: null, + importSource: null, + eMailNotification: false, + scheduled: false, + easyCron: true, + cronString: "* * * * ", + cronMode: "", + minutes: null, + month: null, + hour: null, + day: null, + dayOfMonth: null, + simple: null, + url: null, + insecure: false, + triggerActive: true, + featureType: null, + sortBy: null, + username: "", + password: "", + LOS: 1, + minWidth: null, + maxWidth: null, + depth: null, + sourceOrganization: null + }; +}; + +// initial state +const init = () => { + return { + schedules: [], + importScheduleDetailVisible: false, + currentSchedule: initializeCurrentSchedule() + }; +}; + +const importschedule = { + init, + namespaced: true, + state: init(), + mutations: { + clearCurrentSchedule: state => { + state.currentSchedule = initializeCurrentSchedule(); + }, + setImportScheduleDetailInvisible: state => { + state.importScheduleDetailVisible = false; + }, + setImportScheduleDetailVisible: state => { + state.importScheduleDetailVisible = true; + }, + setSchedules: (state, schedules) => { + state.schedules = schedules; + }, + unmarshallCurrentSchedule: (state, payload) => { + const { kind, config, id } = payload; + const eMailNotification = config["send-email"]; + const { cron, url } = config; + Vue.set(state.currentSchedule, "import_", KINDIMPORTTYPE[kind]); + Vue.set(state.currentSchedule, "id", id); + if (cron) { + Vue.set(state.currentSchedule, "scheduled", true); + Vue.set(state.currentSchedule, "easyCron", false); + Vue.set(state.currentSchedule, "cronString", cron); + } + if (eMailNotification) { + Vue.set(state.currentSchedule, "eMailNotification", eMailNotification); + } + if (url) { + Vue.set(state.currentSchedule, "url", url); + } + let { insecure, username, password, los, depth } = config; + let sortBy = config["sort-by"]; + let minWidth = config["min-width"]; + let maxWidth = config["max-width"]; + let sourceOrganization = config["source-organization"]; + const featureType = config["feature-type"]; + insecure = insecure == "true"; + if (insecure) { + Vue.set(state.currentSchedule, "insecure", insecure); + } + if (featureType) { + Vue.set(state.currentSchedule, "featureType", featureType); + } + if (sortBy) { + Vue.set(state.currentSchedule, "sortBy", sortBy); + } + if (username) { + Vue.set(state.currentSchedule, "username", username); + } + if (password) { + Vue.set(state.currentSchedule, "password", password); + } + if (los) { + Vue.set(state.currentSchedule, "LOS", los); + } + if (minWidth) { + Vue.set(state.currentSchedule, "minWidth", minWidth); + } + if (maxWidth) { + Vue.set(state.currentSchedule, "maxWidth", maxWidth); + } + if (depth) { + Vue.set(state.currentSchedule, "depth", depth); + } + if (sourceOrganization) { + Vue.set( + state.currentSchedule, + "sourceOrganization", + sourceOrganization + ); + } + } + }, + actions: { + loadSchedule({ commit }, id) { + return new Promise((resolve, reject) => { + HTTP.get("/imports/config/" + id, { + headers: { "X-Gemma-Auth": localStorage.getItem("token") } + }) + .then(response => { + commit("unmarshallCurrentSchedule", response.data); + resolve(response); + }) + .catch(error => { + reject(error); + }); + }); + }, + updateCurrentSchedule({ commit }, payload) { + const { data, id } = payload; + return new Promise((resolve, reject) => { + HTTP.patch("imports/config/" + id, data, { + headers: { + "X-Gemma-Auth": localStorage.getItem("token") + } + }) + .then(response => { + resolve(response); + }) + .catch(error => { + reject(error); + }); + }); + }, + saveCurrentSchedule({ commit }, data) { + return new Promise((resolve, reject) => { + HTTP.post("imports/config", data, { + headers: { + "X-Gemma-Auth": localStorage.getItem("token") + } + }) + .then(response => { + resolve(response); + }) + .catch(error => { + reject(error); + }); + }); + }, + loadSchedules({ commit }) { + return new Promise((resolve, reject) => { + HTTP.get("/imports/config", { + headers: { "X-Gemma-Auth": localStorage.getItem("token") } + }) + .then(response => { + commit("setSchedules", response.data); + resolve(response); + }) + .catch(error => { + reject(error); + }); + }); + }, + deleteSchedule({ commit }, id) { + return new Promise((resolve, reject) => { + HTTP.delete("imports/config/" + id, { + headers: { + "X-Gemma-Auth": localStorage.getItem("token") + } + }) + .then(response => { + resolve(response); + }) + .catch(error => { + reject(error); + }); + }); + }, + triggerImport({ commit }, { type, data }) { + return new Promise((resolve, reject) => { + HTTP.post("imports/" + type, data, { + headers: { + "X-Gemma-Auth": localStorage.getItem("token") + } + }) + .then(response => { + resolve(response); + }) + .catch(error => { + reject(error); + }); + }); + } + } +}; + +export { + importschedule, + initializeCurrentSchedule, + IMPORTTYPES, + IMPORTTYPEKIND +}; diff -r b4ba751e70a1 -r 079a1d35e076 client/src/store/index.js --- a/client/src/store/index.js Tue Jan 29 12:36:45 2019 +0100 +++ b/client/src/store/index.js Wed Jan 30 10:03:20 2019 +0100 @@ -22,6 +22,7 @@ import fairwayprofile from "./fairway"; import bottlenecks from "./bottlenecks"; import { imports } from "./imports"; +import { importschedule } from "./importschedule"; Vue.use(Vuex); @@ -32,6 +33,7 @@ application: application.init(), fairwayprofile: fairwayprofile.init(), imports: imports.init(), + importschedule: importschedule.init(), bottlenecks: bottlenecks.init(), map: map.init(), user: user.init(), @@ -43,6 +45,7 @@ application, fairwayprofile, imports, + importschedule, bottlenecks, map, user, diff -r b4ba751e70a1 -r 079a1d35e076 client/tests/e2e/specs/login.js --- a/client/tests/e2e/specs/login.js Tue Jan 29 12:36:45 2019 +0100 +++ b/client/tests/e2e/specs/login.js Wed Jan 30 10:03:20 2019 +0100 @@ -42,10 +42,10 @@ .setValue("input[id='inputPassword']", "oa2Na2") .click("button[type='submit']") .pause(1000) - .click(".userpic") + .click(".menubutton") .pause(1000) - .assert.elementPresent(".username") - .assert.containsText(".username", "oana") + .assert.elementPresent(".logout") + .assert.containsText(".logout", "oana") .end(); }, "Login oana switch url": browser => { @@ -56,15 +56,15 @@ .setValue("input[id='inputPassword']", "oa2Na2") .click("button[type='submit']") .pause(1000) - .click(".userpic") + .click(".menubutton") .pause(1000) - .assert.elementPresent(".username") - .assert.containsText(".username", "oana") + .assert.elementPresent(".logout") + .assert.containsText(".logout", "oana") .url(process.env.VUE_DEV_SERVER_URL + "#/login") .pause(1000) .url(process.env.VUE_DEV_SERVER_URL + "#/") - .assert.elementPresent(".username") - .assert.containsText(".username", "oana") + .assert.elementPresent(".logout") + .assert.containsText(".logout", "oana") .end(); }, "Login switch user from oana to vanja": browser => { @@ -75,17 +75,17 @@ .setValue("input[id='inputPassword']", "oa2Na2") .click("button[type='submit']") .pause(1000) - .click(".userpic") + .click(".menubutton") .pause(1000) - .assert.elementPresent(".username") - .assert.containsText(".username", "oana") + .assert.elementPresent(".logout") + .assert.containsText(".logout", "oana") .url(process.env.VUE_DEV_SERVER_URL + "#/login") .setValue("input[id='inputUsername']", "sophie") .setValue("input[id='inputPassword']", "so2Phie4") .click("button[type='submit']") .pause(1000) - .assert.elementPresent(".username") - .assert.containsText(".username", "sophie") + .assert.elementPresent(".logout") + .assert.containsText(".logout", "sophie") .end(); } }; diff -r b4ba751e70a1 -r 079a1d35e076 pkg/common/attributes.go --- a/pkg/common/attributes.go Tue Jan 29 12:36:45 2019 +0100 +++ b/pkg/common/attributes.go Wed Jan 30 10:03:20 2019 +0100 @@ -20,9 +20,58 @@ "time" ) -// Attributes is a map of optional key/value attributes -// of a configuration. -type Attributes map[string]string +const ( + TimeFormat = "2006-01-02T15:04:05" + DateFormat = "2006-01-02" +) + +type ( + + // Attributes is a map of optional key/value attributes + // of a configuration. + Attributes map[string]string + + AttributesMarshaler interface { + MarshalAttributes(Attributes) error + } + + AttributesUnmarshaler interface { + UnmarshalAttributes(Attributes) error + } +) + +func (ca Attributes) Marshal(src interface{}) error { + if ca == nil { + return nil + } + var err error + if m, ok := src.(AttributesMarshaler); ok { + err = m.MarshalAttributes(ca) + } + return err +} + +func (ca Attributes) Unmarshal(dst interface{}) error { + if ca == nil { + return nil + } + var err error + if um, ok := dst.(AttributesUnmarshaler); ok { + err = um.UnmarshalAttributes(ca) + } + return err +} + +func (ca Attributes) Delete(key string) bool { + if ca == nil { + return false + } + if _, found := ca[key]; !found { + return false + } + delete(ca, key) + return true +} // Get fetches a value for given key out of the configuration. // If the key was not found the bool component of the return value @@ -35,19 +84,55 @@ return value, found } +func (ca Attributes) Set(key, value string) bool { + if ca == nil { + return false + } + ca[key] = value + return true +} + // Bool returns a bool value for a given key. func (ca Attributes) Bool(key string) bool { s, found := ca.Get(key) return found && strings.ToLower(s) == "true" } +func (ca Attributes) SetBool(key string, value bool) bool { + var v string + if value { + v = "true" + } else { + v = "false" + } + return ca.Set(key, v) +} + +func (ca Attributes) Date(key string) (time.Time, bool) { + s, found := ca.Get(key) + if !found { + return time.Time{}, false + } + d, err := time.Parse(DateFormat, s) + if err != nil { + log.Printf("error: %v\n", err) + return time.Time{}, false + } + return d, true +} + +func (ca Attributes) SetDate(key string, date time.Time) bool { + s := date.Format(DateFormat) + return ca.Set(key, s) +} + // Time gives a time.Time for a given key. func (ca Attributes) Time(key string) (time.Time, bool) { s, found := ca.Get(key) if !found { return time.Time{}, false } - t, err := time.Parse("2006-01-02T15:04:05", s) + t, err := time.Parse(TimeFormat, s) if err != nil { log.Printf("error: %v\n", err) return time.Time{}, false @@ -55,6 +140,12 @@ return t, true } +func (ca Attributes) SetTime(key string, t time.Time) bool { + value := t.Format(TimeFormat) + return ca.Set(key, value) + +} + func (ca Attributes) Int(key string) (int, bool) { s, found := ca.Get(key) if !found { @@ -68,6 +159,11 @@ return i, true } +func (ca Attributes) SetInt(key string, value int) bool { + v := strconv.Itoa(value) + return ca.Set(key, v) +} + func (ca Attributes) Duration(key string) (time.Duration, bool) { s, found := ca.Get(key) if !found { @@ -80,3 +176,8 @@ } return d, true } + +func (ca Attributes) SetDuration(key string, value time.Duration) bool { + v := value.String() + return ca.Set(key, v) +} diff -r b4ba751e70a1 -r 079a1d35e076 pkg/controllers/importconfig.go --- a/pkg/controllers/importconfig.go Tue Jan 29 12:36:45 2019 +0100 +++ b/pkg/controllers/importconfig.go Wed Jan 30 10:03:20 2019 +0100 @@ -14,12 +14,10 @@ package controllers import ( - "context" "database/sql" - "errors" + "encoding/json" "fmt" "net/http" - "sort" "strconv" "github.com/gorilla/mux" @@ -30,57 +28,6 @@ "gemma.intevation.de/gemma/pkg/scheduler" ) -const ( - selectImportConfigurationPrefix = ` -SELECT - id, - username, - kind, - send_email, - cron, - url -FROM import.import_configuration` - - selectImportConfigurationSQL = selectImportConfigurationPrefix + ` -ORDER by id` - - selectImportConfigurationIDSQL = selectImportConfigurationPrefix + ` -WHERE id = $1` - - insertImportConfigurationSQL = ` -INSERT INTO import.import_configuration -(username, kind, cron, send_email, url) -VALUES ($1, $2, $3, $4, $5) -RETURNING id` - - insertImportConfigurationAttributeSQL = ` -INSERT INTO import.import_configuration_attributes -(import_configuration_id, k, v) -VALUES ($1, $2, $3)` - - hasImportConfigurationSQL = ` -SELECT true FROM import.import_configuration -WHERE id = $1` - - deleteImportConfiguationAttributesSQL = ` -DELETE FROM import.import_configuration_attributes -WHERE import_configuration_id = $1` - - deleteImportConfiguationSQL = ` -DELETE FROM import.import_configuration -WHERE id = $1` - - updateImportConfigurationSQL = ` -UPDATE import.import_configuration SET - username = $2, - kind = $3, - cron = $4, - url = $5, - send_email = $6 -WHERE id = $1 -` -) - func runImportConfig( _ interface{}, req *http.Request, @@ -117,85 +64,64 @@ ctx := req.Context() - importConfig := input.(*imports.Config) + raw := input.(*json.RawMessage) id, _ := strconv.ParseInt(mux.Vars(req)["id"], 10, 64) - var tx *sql.Tx - - if tx, err = conn.BeginTx(ctx, nil); err != nil { - return - } - defer tx.Rollback() - - var ( - entry imports.IDConfig - kind string - dummy sql.NullString - url sql.NullString - ) - - err = conn.QueryRowContext(ctx, selectImportConfigurationIDSQL, id).Scan( - &entry.ID, - &entry.User, - &kind, - &entry.SendEMail, - &dummy, - &url, - ) - + var pc *imports.PersistentConfig + pc, err = imports.LoadPersistentConfigContext(ctx, conn, id) switch { case err == sql.ErrNoRows: err = JSONError{ Code: http.StatusNotFound, - Message: fmt.Sprintf("No schedule %d found", id), + Message: fmt.Sprintf("No configuration %d found", id), } return case err != nil: return } - session, _ := auth.GetSession(req) - - entry.SendEMail = importConfig.SendEMail - - // We always take the cron spec from the input. - // If there is no spec remove schedule. - var cron sql.NullString - if importConfig.Cron != nil { - cron = sql.NullString{String: string(*importConfig.Cron), Valid: true} + kind := imports.JobKind(pc.Kind) + ctor := imports.ImportModelForJobKind(kind) + if ctor == nil { + err = JSONError{ + Code: http.StatusInternalServerError, + Message: fmt.Sprintf("No constructor for kind '%s' found", pc.Kind), + } + return } - - if importConfig.URL != nil { - url = sql.NullString{String: *importConfig.URL, Valid: true} - } - - if _, err = tx.ExecContext(ctx, updateImportConfigurationSQL, - id, - session.User, - string(importConfig.Kind), - cron, - url, - importConfig.SendEMail, - ); err != nil { + config := ctor() + if err = json.Unmarshal(*raw, config); err != nil { return } - if importConfig.Attributes != nil { - if _, err = tx.ExecContext(ctx, deleteImportConfiguationAttributesSQL, id); err != nil { - return - } - if err = storeConfigAttributes(ctx, tx, id, importConfig.Attributes); err != nil { - return - } + _, oldCron := pc.Attributes.Get("cron") + + session, _ := auth.GetSession(req) + pc.User = session.User + pc.Attributes = common.Attributes{} + pc.Attributes.Marshal(config) + + cron, newCron := pc.Attributes.Get("cron") + + var tx *sql.Tx + if tx, err = conn.BeginTx(ctx, nil); err != nil { + return + } + defer tx.Rollback() + + if err = pc.UpdateContext(ctx, tx); err != nil { + return } - scheduler.UnbindByID(id) + if oldCron { + scheduler.UnbindByID(id) + } - if cron.Valid { + if newCron { if err = scheduler.BindAction( - string(importConfig.Kind), - cron.String, + string(pc.Kind), + cron, id, ); err != nil { return @@ -226,13 +152,13 @@ id, _ := strconv.ParseInt(mux.Vars(req)["id"], 10, 64) - var entry *imports.IDConfig + var cfg *imports.PersistentConfig - entry, err = imports.LoadIDConfigContext(ctx, conn, id) + cfg, err = imports.LoadPersistentConfigContext(ctx, conn, id) switch { case err != nil: return - case entry == nil: + case cfg == nil: err = JSONError{ Code: http.StatusNotFound, Message: fmt.Sprintf("No schedule %d found", id), @@ -240,7 +166,28 @@ return } - jr = JSONResult{Result: &entry} + kind := imports.JobKind(cfg.Kind) + + ctor := imports.ImportModelForJobKind(kind) + if ctor == nil { + err = JSONError{ + Code: http.StatusInternalServerError, + Message: fmt.Sprintf("No constructor for kind '%s' found", cfg.Kind), + } + return + } + + what := ctor() + + if err = cfg.Attributes.Unmarshal(what); err != nil { + return + } + + jr = JSONResult{Result: &imports.ImportConfigOut{ + ID: id, + Kind: imports.ImportKind(cfg.Kind), + Config: what, + }} return } @@ -260,28 +207,21 @@ } defer tx.Rollback() - var found bool - err = tx.QueryRowContext(ctx, hasImportConfigurationSQL, id).Scan(&found) + err = imports.DeletePersistentConfigurationContext( + ctx, + tx, + id, + ) + switch { case err == sql.ErrNoRows: err = JSONError{ Code: http.StatusNotFound, - Message: fmt.Sprintf("No schedule %d found", id), + Message: fmt.Sprintf("No configuration %d found", id), } return case err != nil: return - case !found: - err = errors.New("Unexpected result") - return - } - - if _, err = tx.ExecContext(ctx, deleteImportConfiguationAttributesSQL, id); err != nil { - return - } - - if _, err = tx.ExecContext(ctx, deleteImportConfiguationSQL, id); err != nil { - return } // Remove from running scheduler. @@ -298,92 +238,58 @@ } jr = JSONResult{Result: &result} + return } -func storeConfigAttributes( - ctx context.Context, - tx *sql.Tx, - id int64, - attrs common.Attributes, -) error { - if len(attrs) == 0 { - return nil - } - attrStmt, err := tx.PrepareContext(ctx, insertImportConfigurationAttributeSQL) - if err != nil { - return err - } - defer attrStmt.Close() - // Sort to make it deterministic - keys := make([]string, len(attrs)) - i := 0 - for key := range attrs { - keys[i] = key - i++ - } - sort.Strings(keys) - for _, key := range keys { - if _, err := attrStmt.ExecContext(ctx, id, key, attrs[key]); err != nil { - return err - } - } - return nil -} - func addImportConfig( input interface{}, req *http.Request, conn *sql.Conn, ) (jr JSONResult, err error) { - importConfig := input.(*imports.Config) + cfg := input.(*imports.ImportConfigIn) + + kind := imports.JobKind(cfg.Kind) + + ctor := imports.ImportModelForJobKind(kind) + if ctor == nil { + err = JSONError{ + Code: http.StatusBadRequest, + Message: fmt.Sprintf("No kind %s found", string(cfg.Kind)), + } + return + } + config := ctor() + if err = json.Unmarshal(cfg.Config, config); err != nil { + return + } session, _ := auth.GetSession(req) - var cron, url sql.NullString - - if importConfig.Cron != nil { - cron = sql.NullString{String: string(*importConfig.Cron), Valid: true} + pc := imports.PersistentConfig{ + User: session.User, + Kind: string(cfg.Kind), + Attributes: common.Attributes{}, } - if importConfig.URL != nil { - url = sql.NullString{String: *importConfig.URL, Valid: true} - } + pc.Attributes.Marshal(config) ctx := req.Context() var tx *sql.Tx - if tx, err = conn.BeginTx(ctx, nil); err != nil { return } defer tx.Rollback() var id int64 - if err = tx.QueryRowContext( - ctx, - insertImportConfigurationSQL, - session.User, - string(importConfig.Kind), - cron, - importConfig.SendEMail, - url, - ).Scan(&id); err != nil { - return - } - - // Store extra attributes - if err = storeConfigAttributes(ctx, tx, id, importConfig.Attributes); err != nil { + if id, err = pc.StoreContext(ctx, tx); err != nil { return } // Need to start a scheduler job right away? - if importConfig.Cron != nil { - if err = scheduler.BindAction( - string(importConfig.Kind), - string(*importConfig.Cron), - id, - ); err != nil { + if cron, ok := pc.Attributes.Get("cron"); ok { + if err = scheduler.BindAction(string(cfg.Kind), cron, id); err != nil { return } } @@ -413,47 +319,17 @@ ) (jr JSONResult, err error) { ctx := req.Context() - var rows *sql.Rows + configs := []*imports.ImportConfigOut{} - if rows, err = conn.QueryContext(ctx, selectImportConfigurationSQL); err != nil { + if err = imports.ListAllPersistentConfigurationsContext( + ctx, conn, + func(config *imports.ImportConfigOut) error { + configs = append(configs, config) + return nil + }, + ); err != nil { return } - defer rows.Close() - - list := []*imports.IDConfig{} - - for rows.Next() { - var ( - entry imports.IDConfig - kind string - cron sql.NullString - url sql.NullString - ) - if err = rows.Scan( - &entry.ID, - &entry.User, - &kind, - &entry.SendEMail, - &cron, - &url, - ); err != nil { - return - } - entry.Kind = imports.ImportKind(kind) - if cron.Valid { - cs := imports.CronSpec(cron.String) - entry.Cron = &cs - } - if url.Valid { - entry.URL = &url.String - } - list = append(list, &entry) - } - - if err = rows.Err(); err != nil { - return - } - - jr = JSONResult{Result: list} + jr = JSONResult{Result: configs} return } diff -r b4ba751e70a1 -r 079a1d35e076 pkg/controllers/json.go --- a/pkg/controllers/json.go Tue Jan 29 12:36:45 2019 +0100 +++ b/pkg/controllers/json.go Wed Jan 30 10:03:20 2019 +0100 @@ -44,7 +44,7 @@ type JSONHandler struct { // Input (if not nil) is called to fill a data structure // returned by this function. - Input func() interface{} + Input func(*http.Request) interface{} // Handle is called to handle the incoming HTTP request. // in is the data structure returned by Input. Its nil if Input is nil. // req is the incoming HTTP request. @@ -79,7 +79,7 @@ var input interface{} if j.Input != nil { - input = j.Input() + input = j.Input(req) defer req.Body.Close() var r io.Reader switch { diff -r b4ba751e70a1 -r 079a1d35e076 pkg/controllers/manualimports.go --- a/pkg/controllers/manualimports.go Tue Jan 29 12:36:45 2019 +0100 +++ b/pkg/controllers/manualimports.go Wed Jan 30 10:03:20 2019 +0100 @@ -24,182 +24,88 @@ "gemma.intevation.de/gemma/pkg/common" "gemma.intevation.de/gemma/pkg/imports" "gemma.intevation.de/gemma/pkg/models" + "github.com/gorilla/mux" ) -func importBottleneck(input interface{}) (interface{}, common.Attributes, bool) { - bi := input.(*models.BottleneckImport) - bn := &imports.Bottleneck{ - URL: bi.URL, - Insecure: bi.Insecure, - } - return bn, bi.Attributes, bi.SendEmail -} - -func importGaugeMeasurement(input interface{}) (interface{}, common.Attributes, bool) { - gi := input.(*models.GaugeMeasurementImport) - gm := &imports.GaugeMeasurement{ - URL: gi.URL, - Insecure: gi.Insecure, - } - return gm, gi.Attributes, gi.SendEmail -} - -func importFairwayAvailability(input interface{}) (interface{}, common.Attributes, bool) { - fai := input.(*models.FairwayAvailabilityImport) - fa := &imports.FairwayAvailability{ - URL: fai.URL, - Insecure: fai.Insecure, - } - return fa, fai.Attributes, fai.SendEmail -} - -func importWaterwayAxis(input interface{}) (interface{}, common.Attributes, bool) { - wxi := input.(*models.WaterwayAxisImport) - wx := &imports.WaterwayAxis{ - URL: wxi.URL, - FeatureType: wxi.FeatureType, - SortBy: wxi.SortBy, - } - return wx, wxi.Attributes, wxi.SendEmail -} - -func importWaterwayArea(input interface{}) (interface{}, common.Attributes, bool) { - wai := input.(*models.WaterwayAreaImport) - wa := &imports.WaterwayArea{ - URL: wai.URL, - FeatureType: wai.FeatureType, - SortBy: wai.SortBy, - } - return wa, wai.Attributes, wai.SendEmail -} - -func importWaterwayGauge(input interface{}) (interface{}, common.Attributes, bool) { - wgi := input.(*models.WaterwayGaugeImport) - username, _ := wgi.Attributes.Get("username") - password, _ := wgi.Attributes.Get("password") - insecure := wgi.Attributes.Bool("insecure") - wg := &imports.WaterwayGauge{ - URL: wgi.URL, - Username: username, - Password: password, - Insecure: insecure, +func importModel(req *http.Request) interface{} { + kind := mux.Vars(req)["kind"] + ctor := imports.ImportModelForJobKind(imports.JobKind(kind)) + if ctor == nil { + log.Printf("error: Unknown job kind '%s'.\n", kind) + panic(http.ErrAbortHandler) } - return wg, wgi.Attributes, wgi.SendEmail -} - -func importDistancemarksVirtual(input interface{}) (interface{}, common.Attributes, bool) { - dmvi := input.(*models.DistanceMarksVirtualImport) - username, _ := dmvi.Attributes.Get("username") - password, _ := dmvi.Attributes.Get("password") - insecure := dmvi.Attributes.Bool("insecure") - wg := &imports.DistanceMarksVirtual{ - URL: dmvi.URL, - Username: username, - Password: password, - Insecure: insecure, - } - return wg, dmvi.Attributes, dmvi.SendEmail -} - -func importFairwayDimension(input interface{}) (interface{}, common.Attributes, bool) { - fdi := input.(*models.FairwayDimensionImport) - fd := &imports.FairwayDimension{ - URL: fdi.URL, - FeatureType: fdi.FeatureType, - SortBy: fdi.SortBy, - LOS: fdi.LOS, - MinWidth: fdi.MinWidth, - MaxWidth: fdi.MaxWidth, - Depth: fdi.Depth, - SourceOrganization: fdi.SourceOrganization, - } - return fd, fdi.Attributes, fdi.SendEmail -} - -func importDistanceMarksAshore(input interface{}) (interface{}, common.Attributes, bool) { - dmai := input.(*models.DistanceMarksAshoreImport) - dma := &imports.DistanceMarksAshore{ - URL: dmai.URL, - FeatureType: dmai.FeatureType, - SortBy: dmai.SortBy, - } - return dma, dmai.Attributes, dmai.SendEmail -} - -func importStretch(input interface{}) (interface{}, common.Attributes, bool) { - sti := input.(*models.StretchImport) - st := &imports.Stretch{ - Name: sti.Name, - From: sti.From, - To: sti.To, - ObjNam: sti.ObjNam, - NObjNam: sti.NObjNam, - Source: sti.Source, - Date: sti.Date, - Countries: sti.Countries, - } - return st, sti.Attributes, sti.SendEmail -} - -func retry(a common.Attributes) (time.Time, *int, *time.Duration) { - due, _ := a.Time("due") - ret, ok := a.Int("retries") - var retries *int - if ok { - retries = &ret - } - dur, ok := a.Duration("wait-retry") - var duration *time.Duration - if ok { - duration = &dur - } - return due, retries, duration + return ctor() } func manualImport( - kind imports.JobKind, - setup func(interface{}) (interface{}, common.Attributes, bool), -) func(interface{}, *http.Request, *sql.Conn) (JSONResult, error) { - - return func(input interface{}, req *http.Request, _ *sql.Conn) ( - jr JSONResult, err error) { - - what, attrs, sendEmail := setup(input) - - due, retries, waitRetry := retry(attrs) - - var serialized string - if serialized, err = common.ToJSONString(what); err != nil { - return - } - - session, _ := auth.GetSession(req) + input interface{}, + req *http.Request, + _ *sql.Conn, +) (jr JSONResult, err error) { - var jobID int64 - if jobID, err = imports.AddJob( - kind, - due, - retries, - waitRetry, - session.User, - sendEmail, - serialized, - ); err != nil { - return - } - - log.Printf("info: added import #%d to queue\n", jobID) - - result := struct { - ID int64 `json:"id"` - }{ - ID: jobID, - } - - jr = JSONResult{ - Code: http.StatusCreated, - Result: &result, + kind := imports.JobKind(mux.Vars(req)["kind"]) + what := imports.ConvertToInternal(kind, input) + if what == nil { + err = JSONError{ + Code: http.StatusInternalServerError, + Message: "Unable to convert import models", } return } + + var serialized string + if serialized, err = common.ToJSONString(what); err != nil { + return + } + + var ( + due time.Time + trys *int + waitRetry *time.Duration + email bool + ) + + if qctg, ok := input.(models.QueueConfigurationGetter); ok { + qct := qctg.GetQueueConfiguration() + if qct.Due != nil { + due = qct.Due.Time + } + trys = qct.Trys + if qct.WaitRetry != nil { + waitRetry = &qct.WaitRetry.Duration + } + } + + if etg, ok := input.(models.EmailTypeGetter); ok { + email = etg.GetEmailType().Email + } + + session, _ := auth.GetSession(req) + + var jobID int64 + if jobID, err = imports.AddJob( + kind, + due, + trys, + waitRetry, + session.User, + email, + serialized, + ); err != nil { + return + } + + log.Printf("info: added import #%d to queue\n", jobID) + + result := struct { + ID int64 `json:"id"` + }{ + ID: jobID, + } + + jr = JSONResult{ + Code: http.StatusCreated, + Result: &result, + } + return } diff -r b4ba751e70a1 -r 079a1d35e076 pkg/controllers/routes.go --- a/pkg/controllers/routes.go Tue Jan 29 12:36:45 2019 +0100 +++ b/pkg/controllers/routes.go Wed Jan 30 10:03:20 2019 +0100 @@ -15,8 +15,10 @@ package controllers import ( + "encoding/json" "net/http" "net/http/httputil" + "strings" "github.com/gorilla/mux" @@ -43,7 +45,7 @@ })).Methods(http.MethodGet) api.Handle("/users", sysAdmin(&JSONHandler{ - Input: func() interface{} { return new(models.User) }, + Input: func(*http.Request) interface{} { return new(models.User) }, Handle: createUser, })).Methods(http.MethodPost) @@ -52,7 +54,7 @@ })).Methods(http.MethodGet) api.Handle("/users/{user}", any(&JSONHandler{ - Input: func() interface{} { return new(models.User) }, + Input: func(*http.Request) interface{} { return new(models.User) }, Handle: updateUser, })).Methods(http.MethodPut) @@ -77,13 +79,13 @@ })).Methods(http.MethodGet) api.Handle("/system/style/{feature}/{attr}", any(&JSONHandler{ - Input: func() interface{} { return new(models.Colour) }, + Input: func(*http.Request) interface{} { return new(models.Colour) }, Handle: setFeatureStyle, })).Methods(http.MethodPut) // Password resets. api.Handle("/users/passwordreset", &JSONHandler{ - Input: func() interface{} { return new(models.PWResetUser) }, + Input: func(*http.Request) interface{} { return new(models.PWResetUser) }, Handle: passwordResetRequest, NoConn: true, }).Methods(http.MethodPost) @@ -147,13 +149,13 @@ // Cross sections api.Handle("/cross", any(&JSONHandler{ - Input: func() interface{} { return new(models.CrossSectionInput) }, + Input: func(*http.Request) interface{} { return new(models.CrossSectionInput) }, Handle: crossSection, })).Methods(http.MethodPost) // Feature search api.Handle("/search", any(&JSONHandler{ - Input: func() interface{} { return new(models.SearchRequest) }, + Input: func(*http.Request) interface{} { return new(models.SearchRequest) }, Handle: searchFeature, })).Methods(http.MethodPost) @@ -162,76 +164,33 @@ sysAdmin(http.HandlerFunc(uploadStyle))).Methods(http.MethodPost) // Imports - api.Handle("/imports/soundingresult-upload/{token}", + api.Handle("/imports/sr-upload/{token}", waterwayAdmin(http.HandlerFunc(deleteSoundingUpload))).Methods(http.MethodDelete) - api.Handle("/imports/soundingresult-upload", waterwayAdmin(&JSONHandler{ + api.Handle("/imports/sr-upload", waterwayAdmin(&JSONHandler{ Handle: uploadSoundingResult, })).Methods(http.MethodPost) - api.Handle("/imports/soundingresult", waterwayAdmin( + api.Handle("/imports/sr", waterwayAdmin( http.HandlerFunc(importSoundingResult))).Methods(http.MethodPost) - api.Handle("/imports/approvedgm", waterwayAdmin( + api.Handle("/imports/agm", waterwayAdmin( http.HandlerFunc(importApprovedGaugeMeasurements))).Methods(http.MethodPost) - api.Handle("/imports/bottleneck", waterwayAdmin(&JSONHandler{ - Input: func() interface{} { return new(models.BottleneckImport) }, - Handle: manualImport(imports.BNJobKind, importBottleneck), - NoConn: true, - })).Methods(http.MethodPost) - - api.Handle("/imports/gaugemeasurement", waterwayAdmin(&JSONHandler{ - Input: func() interface{} { return new(models.GaugeMeasurementImport) }, - Handle: manualImport(imports.GMJobKind, importGaugeMeasurement), - NoConn: true, - })).Methods(http.MethodPost) - - api.Handle("/imports/fairwayavailability", waterwayAdmin(&JSONHandler{ - Input: func() interface{} { return new(models.FairwayAvailabilityImport) }, - Handle: manualImport(imports.FAJobKind, importFairwayAvailability), + api.Handle("/imports/{kind:st}", sysAdmin(&JSONHandler{ + Input: importModel, + Handle: manualImport, NoConn: true, })).Methods(http.MethodPost) - api.Handle("/imports/waterwayaxis", waterwayAdmin(&JSONHandler{ - Input: func() interface{} { return new(models.WaterwayAxisImport) }, - Handle: manualImport(imports.WXJobKind, importWaterwayAxis), - NoConn: true, - })).Methods(http.MethodPost) - - api.Handle("/imports/waterwayarea", waterwayAdmin(&JSONHandler{ - Input: func() interface{} { return new(models.WaterwayAreaImport) }, - Handle: manualImport(imports.WAJobKind, importWaterwayArea), - NoConn: true, - })).Methods(http.MethodPost) - - api.Handle("/imports/waterwaygauge", waterwayAdmin(&JSONHandler{ - Input: func() interface{} { return new(models.WaterwayGaugeImport) }, - Handle: manualImport(imports.WGJobKind, importWaterwayGauge), - NoConn: true, - })).Methods(http.MethodPost) + kinds := strings.Join([]string{ + "bn", "gm", "fa", "wx", "wa", + "wg", "dmv", "fd", "dm", + }, "|") - api.Handle("/imports/distancemarksvirtual", waterwayAdmin(&JSONHandler{ - Input: func() interface{} { return new(models.DistanceMarksVirtualImport) }, - Handle: manualImport(imports.DMVJobKind, importDistancemarksVirtual), - NoConn: true, - })).Methods(http.MethodPost) - - api.Handle("/imports/fairwaydimension", waterwayAdmin(&JSONHandler{ - Input: func() interface{} { return new(models.FairwayDimensionImport) }, - Handle: manualImport(imports.FDJobKind, importFairwayDimension), - NoConn: true, - })).Methods(http.MethodPost) - - api.Handle("/imports/distancemarks", waterwayAdmin(&JSONHandler{ - Input: func() interface{} { return new(models.DistanceMarksAshoreImport) }, - Handle: manualImport(imports.DMAJobKind, importDistanceMarksAshore), - NoConn: true, - })).Methods(http.MethodPost) - - api.Handle("/imports/stretch", sysAdmin(&JSONHandler{ - Input: func() interface{} { return new(models.StretchImport) }, - Handle: manualImport(imports.STJobKind, importStretch), + api.Handle("/imports/{kind:"+kinds+"}", waterwayAdmin(&JSONHandler{ + Input: importModel, + Handle: manualImport, NoConn: true, })).Methods(http.MethodPost) @@ -243,7 +202,7 @@ api.Handle("/imports/config/{id:[0-9]+}", waterwayAdmin(&JSONHandler{ - Input: func() interface{} { return new(imports.Config) }, + Input: func(*http.Request) interface{} { return &json.RawMessage{} }, Handle: modifyImportConfig, })).Methods(http.MethodPatch) @@ -259,7 +218,7 @@ api.Handle("/imports/config", waterwayAdmin(&JSONHandler{ - Input: func() interface{} { return new(imports.Config) }, + Input: func(*http.Request) interface{} { return new(imports.ImportConfigIn) }, Handle: addImportConfig, })).Methods(http.MethodPost) @@ -281,7 +240,7 @@ })).Methods(http.MethodGet) api.Handle("/imports", waterwayAdmin(&JSONHandler{ - Input: func() interface{} { return &[]models.Review{} }, + Input: func(*http.Request) interface{} { return &[]models.Review{} }, Handle: reviewImports, })).Methods(http.MethodPatch) diff -r b4ba751e70a1 -r 079a1d35e076 pkg/controllers/srimports.go --- a/pkg/controllers/srimports.go Tue Jan 29 12:36:45 2019 +0100 +++ b/pkg/controllers/srimports.go Wed Jan 30 10:03:20 2019 +0100 @@ -116,7 +116,7 @@ } if v := req.FormValue("date"); v != "" { - date, err := time.Parse(models.DateFormat, v) + date, err := time.Parse(common.DateFormat, v) if err != nil { return err } diff -r b4ba751e70a1 -r 079a1d35e076 pkg/imports/config.go --- a/pkg/imports/config.go Tue Jan 29 12:36:45 2019 +0100 +++ b/pkg/imports/config.go Wed Jan 30 10:03:20 2019 +0100 @@ -18,49 +18,34 @@ "database/sql" "encoding/json" "fmt" - - "github.com/robfig/cron" + "sort" "gemma.intevation.de/gemma/pkg/auth" "gemma.intevation.de/gemma/pkg/common" ) type ( - // CronSpec is a string containing a cron line. - CronSpec string - // ImportKind is a string which has to be one // of the registered import types. ImportKind string - // Config is JSON serialized form of a import configuration. - Config struct { - // Kind is the import type. - Kind ImportKind `json:"kind"` - // SendEMail indicates if a mail should be be send - // when the import was changed to states - // 'pending' or 'failed'. - SendEMail bool `json:"send-email"` - // Cron is the cron schedule - // of this configuration if this value is not - // nil. If nil the import is not scheduled. - Cron *CronSpec `json:"cron"` - // URL is an optional URL used by the import. - URL *string `json:"url"` - // Attributes are optional key/value pairs for a configuration. - Attributes common.Attributes `json:"attributes,omitempty"` + ImportConfigIn struct { + Kind ImportKind `json:"kind"` + Config json.RawMessage `json:"config"` } - // IDConfig is the same as Config with an ID. - // Mainly used for server delivered configurations. - IDConfig struct { - ID int64 `json:"id"` - User string `json:"user"` - Kind ImportKind `json:"kind"` - SendEMail bool `json:"send-email"` - Cron *CronSpec `json:"cron,omitempty"` - URL *string `json:"url,omitempty"` - Attributes common.Attributes `json:"attributes,omitempty"` + ImportConfigOut struct { + ID int64 `json:"id"` + Kind ImportKind `json:"kind"` + User string `json:"user"` + Config interface{} `json:"config,omitempty"` + } + + PersistentConfig struct { + ID int64 + User string + Kind string + Attributes common.Attributes } ) @@ -81,52 +66,92 @@ return nil } -// UnmarshalJSON checks if the incoming string is -// a valid cron line. -func (cs *CronSpec) UnmarshalJSON(data []byte) error { - var spec string - if err := json.Unmarshal(data, &spec); err != nil { - return err - } - if _, err := cron.Parse(spec); err != nil { - return err - } - *cs = CronSpec(spec) - - return nil -} - const ( configUser = "sys_admin" - loadConfigSQL = ` + loadPersistentConfigSQL = ` SELECT username, - kind, - send_email, - cron, - url + kind FROM import.import_configuration WHERE id = $1` - loadConfigAttributesSQL = ` + loadPersistentConfigAttributesSQL = ` SELECT k, v FROM import.import_configuration_attributes WHERE import_configuration_id = $1` + + hasImportConfigurationSQL = ` +SELECT true FROM import.import_configuration +WHERE id = $1` + + deleteImportConfiguationAttributesSQL = ` +DELETE FROM import.import_configuration_attributes +WHERE import_configuration_id = $1` + + deleteImportConfiguationSQL = ` +DELETE FROM import.import_configuration +WHERE id = $1` + + updateImportConfigurationSQL = ` +UPDATE import.import_configuration SET + username = $2, + kind = $3 +WHERE id = $1` + + selectImportConfigurationsByID = ` +SELECT + c.id AS id, + username, + kind, + a.k, + a.v +FROM import.import_configuration c JOIN + import.import_configuration_attributes a + ON c.id = a.import_configuration_id +ORDER by c.id` + + insertImportConfigurationSQL = ` +INSERT INTO import.import_configuration +(username, kind) +VALUES ($1, $2) +RETURNING id` + + insertImportConfigurationAttributeSQL = ` +INSERT INTO import.import_configuration_attributes +(import_configuration_id, k, v) +VALUES ($1, $2, $3)` ) -// LoadIDConfigContext loads an import configuration from database. -func LoadIDConfigContext(ctx context.Context, conn *sql.Conn, id int64) (*IDConfig, error) { +func (pc *PersistentConfig) UpdateContext(ctx context.Context, tx *sql.Tx) error { + if _, err := tx.ExecContext( + ctx, + updateImportConfigurationSQL, + pc.ID, pc.User, pc.Kind, + ); err != nil { + return err + } + if _, err := tx.ExecContext( + ctx, + deleteImportConfiguationAttributesSQL, + pc.ID, + ); err != nil { + return err + } + return storeConfigAttributes(ctx, tx, pc.ID, pc.Attributes) +} - cfg := &IDConfig{ID: id} - var kind ImportKind - var cron, url sql.NullString - err := conn.QueryRowContext(ctx, loadConfigSQL, id).Scan( +func LoadPersistentConfigContext( + ctx context.Context, + conn *sql.Conn, + id int64, +) (*PersistentConfig, error) { + + cfg := &PersistentConfig{ID: id} + + err := conn.QueryRowContext(ctx, loadPersistentConfigSQL, id).Scan( &cfg.User, - &kind, - &cfg.SendEMail, - &cron, - &url, + &cfg.Kind, ) switch { @@ -136,16 +161,8 @@ return nil, err } - cfg.Kind = ImportKind(kind) - if cron.Valid { - c := CronSpec(cron.String) - cfg.Cron = &c - } - if url.Valid { - cfg.URL = &url.String - } // load the extra attributes. - rows, err := conn.QueryContext(ctx, loadConfigAttributesSQL, id) + rows, err := conn.QueryContext(ctx, loadPersistentConfigAttributesSQL, id) if err != nil { return nil, err } @@ -170,16 +187,171 @@ return cfg, nil } -func loadIDConfig(id int64) (*IDConfig, error) { - return loadIDConfigContext(context.Background(), id) +func loadPersistentConfig(id int64) (*PersistentConfig, error) { + return loadPersistentConfigContext(context.Background(), id) } -func loadIDConfigContext(ctx context.Context, id int64) (*IDConfig, error) { - var cfg *IDConfig +func loadPersistentConfigContext(ctx context.Context, id int64) (*PersistentConfig, error) { + var cfg *PersistentConfig err := auth.RunAs(ctx, configUser, func(conn *sql.Conn) error { var err error - cfg, err = LoadIDConfigContext(ctx, conn, id) + cfg, err = LoadPersistentConfigContext(ctx, conn, id) return err }) return cfg, err } + +func ListAllPersistentConfigurationsContext( + ctx context.Context, + conn *sql.Conn, + fn func(*ImportConfigOut) error, +) error { + + rows, err := conn.QueryContext(ctx, selectImportConfigurationsByID) + if err != nil { + return err + } + defer rows.Close() + + var ( + first = true + lastID int64 + pc PersistentConfig + k, v sql.NullString + ) + + send := func() error { + kind := JobKind(pc.Kind) + ctor := ImportModelForJobKind(kind) + if ctor == nil { + return fmt.Errorf("unable to deserialize kind '%s'", pc.Kind) + } + config := ctor() + pc.Attributes.Unmarshal(config) + return fn(&ImportConfigOut{ + ID: pc.ID, + Kind: ImportKind(pc.Kind), + User: pc.User, + Config: config, + }) + } + + for rows.Next() { + if err := rows.Scan( + &pc.ID, + &pc.User, + &pc.Kind, + &k, &v, + ); err != nil { + return err + } + if !first { + if lastID != pc.ID { + if err := send(); err != nil { + return err + } + pc.Attributes = nil + } + } else { + first = false + } + + if k.Valid && v.Valid { + if pc.Attributes == nil { + pc.Attributes = common.Attributes{} + } + pc.Attributes.Set(k.String, v.String) + } + + lastID = pc.ID + } + + if err := rows.Err(); err != nil { + return err + } + + err = nil + if !first { + err = send() + } + return err +} + +func DeletePersistentConfigurationContext( + ctx context.Context, + tx *sql.Tx, + id int64, +) error { + var found bool + if err := tx.QueryRowContext( + ctx, + hasImportConfigurationSQL, + id, + ).Scan(&found); err != nil { + return err + } + if !found { + return sql.ErrNoRows + } + if _, err := tx.ExecContext( + ctx, + deleteImportConfiguationAttributesSQL, + id, + ); err != nil { + return nil + } + _, err := tx.ExecContext( + ctx, + deleteImportConfiguationSQL, + id, + ) + return err +} + +func storeConfigAttributes( + ctx context.Context, + tx *sql.Tx, + id int64, + attrs common.Attributes, +) error { + if len(attrs) == 0 { + return nil + } + attrStmt, err := tx.PrepareContext(ctx, insertImportConfigurationAttributeSQL) + if err != nil { + return err + } + defer attrStmt.Close() + // Sort to make it deterministic + keys := make([]string, len(attrs)) + i := 0 + for key := range attrs { + keys[i] = key + i++ + } + sort.Strings(keys) + for _, key := range keys { + if _, err := attrStmt.ExecContext(ctx, id, key, attrs[key]); err != nil { + return err + } + } + return nil +} + +func (pc *PersistentConfig) StoreContext(ctx context.Context, tx *sql.Tx) (int64, error) { + var id int64 + if err := tx.QueryRowContext( + ctx, + insertImportConfigurationSQL, + pc.User, + pc.Kind, + ).Scan(&id); err != nil { + return 0, err + } + + if err := storeConfigAttributes(ctx, tx, id, pc.Attributes); err != nil { + return 0, err + } + pc.ID = id + return id, nil +} diff -r b4ba751e70a1 -r 079a1d35e076 pkg/imports/fa.go --- a/pkg/imports/fa.go Tue Jan 29 12:36:45 2019 +0100 +++ b/pkg/imports/fa.go Wed Jan 30 10:03:20 2019 +0100 @@ -22,7 +22,6 @@ "github.com/jackc/pgx/pgtype" "gemma.intevation.de/gemma/pkg/common" - "gemma.intevation.de/gemma/pkg/models" "gemma.intevation.de/gemma/pkg/soap/ifaf" ) @@ -36,6 +35,11 @@ Insecure bool `json:"insecure"` } +type uniqueFairwayAvailability struct { + BottleneckId string + Surdat time.Time +} + // FAJobKind is import queue type identifier. const FAJobKind JobKind = "fa" @@ -182,6 +186,11 @@ // CleanUp of a fairway availablities import is a NOP. func (*FairwayAvailability) CleanUp() error { return nil } +type bottleneckCountry struct { + ID string + ResponsibleCountry string +} + // Do executes the actual fairway availability import. func (fa *FairwayAvailability) Do( ctx context.Context, @@ -199,10 +208,10 @@ } defer rows.Close() - bottlenecks := []models.Bottleneck{} + bottlenecks := []bottleneckCountry{} for rows.Next() { - var bn models.Bottleneck + var bn bottleneckCountry if err = rows.Scan( &bn.ID, &bn.ResponsibleCountry, @@ -220,7 +229,7 @@ if err != nil { return nil, err } - fairwayAvailabilities := map[models.UniqueFairwayAvailability]int64{} + fairwayAvailabilities := map[uniqueFairwayAvailability]int64{} for faRows.Next() { var id int64 var bnId string @@ -232,7 +241,7 @@ ); err != nil { return nil, err } - key := models.UniqueFairwayAvailability{ + key := uniqueFairwayAvailability{ BottleneckId: bnId, Surdat: sd, } @@ -274,8 +283,8 @@ func (fa *FairwayAvailability) doForFAs( ctx context.Context, - bottlenecks []models.Bottleneck, - fairwayAvailabilities map[models.UniqueFairwayAvailability]int64, + bottlenecks []bottleneckCountry, + fairwayAvailabilities map[uniqueFairwayAvailability]int64, latestDate pgtype.Timestamp, conn *sql.Conn, feedback Feedback, @@ -344,7 +353,7 @@ var faID int64 feedback.Info("Found %d fairway availabilities", len(result.FairwayAvailability)) for _, faRes := range result.FairwayAvailability { - uniqueFa := models.UniqueFairwayAvailability{ + uniqueFa := uniqueFairwayAvailability{ BottleneckId: faRes.Bottleneck_id, Surdat: faRes.SURDAT, } diff -r b4ba751e70a1 -r 079a1d35e076 pkg/imports/gm.go --- a/pkg/imports/gm.go Tue Jan 29 12:36:45 2019 +0100 +++ b/pkg/imports/gm.go Wed Jan 30 10:03:20 2019 +0100 @@ -35,6 +35,12 @@ Insecure bool `json:"insecure"` } +// gaugeMeasurement holds information about a gauge and the latest measurement +type gaugeMeasurement struct { + Gauge models.Isrs + LatestDateIssue time.Time +} + // GMJobKind is the import queue type identifier. const GMJobKind JobKind = "gm" @@ -157,10 +163,10 @@ } defer rows.Close() - gauges := []models.GaugeMeasurement{} + gauges := []gaugeMeasurement{} for rows.Next() { - var g models.GaugeMeasurement + var g gaugeMeasurement if err = rows.Scan( &g.Gauge.CountryCode, &g.Gauge.LoCode, @@ -224,7 +230,7 @@ func (gm *GaugeMeasurement) doForGM( ctx context.Context, - gauges []models.GaugeMeasurement, + gauges []gaugeMeasurement, conn *sql.Conn, feedback Feedback, ) ([]string, error) { diff -r b4ba751e70a1 -r 079a1d35e076 pkg/imports/modelconvert.go --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pkg/imports/modelconvert.go Wed Jan 30 10:03:20 2019 +0100 @@ -0,0 +1,152 @@ +// This is Free Software under GNU Affero General Public License v >= 3.0 +// without warranty, see README.md and license for details. +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// License-Filename: LICENSES/AGPL-3.0.txt +// +// Copyright (C) 2018 by via donau +// – Österreichische Wasserstraßen-Gesellschaft mbH +// Software engineering by Intevation GmbH +// +// Author(s): +// * Sascha L. Teichmann + +package imports + +import ( + "gemma.intevation.de/gemma/pkg/models" +) + +var kindToImportModel = map[JobKind]func() interface{}{ + BNJobKind: func() interface{} { return new(models.BottleneckImport) }, + GMJobKind: func() interface{} { return new(models.GaugeMeasurementImport) }, + FAJobKind: func() interface{} { return new(models.FairwayAvailabilityImport) }, + WXJobKind: func() interface{} { return new(models.WaterwayAxisImport) }, + WAJobKind: func() interface{} { return new(models.WaterwayAreaImport) }, + WGJobKind: func() interface{} { return new(models.WaterwayGaugeImport) }, + DMVJobKind: func() interface{} { return new(models.DistanceMarksVirtualImport) }, + FDJobKind: func() interface{} { return new(models.FairwayDimensionImport) }, + DMAJobKind: func() interface{} { return new(models.DistanceMarksAshoreImport) }, + STJobKind: func() interface{} { return new(models.StretchImport) }, +} + +func ImportModelForJobKind(kind JobKind) func() interface{} { + return kindToImportModel[kind] +} + +var convertModel = map[JobKind]func(interface{}) interface{}{ + + BNJobKind: func(input interface{}) interface{} { + bi := input.(*models.BottleneckImport) + return &Bottleneck{ + URL: bi.URL, + Insecure: bi.Insecure, + } + }, + + GMJobKind: func(input interface{}) interface{} { + gi := input.(*models.GaugeMeasurementImport) + return &GaugeMeasurement{ + URL: gi.URL, + Insecure: gi.Insecure, + } + }, + + FAJobKind: func(input interface{}) interface{} { + fai := input.(*models.FairwayAvailabilityImport) + return &FairwayAvailability{ + URL: fai.URL, + Insecure: fai.Insecure, + } + }, + + WXJobKind: func(input interface{}) interface{} { + wxi := input.(*models.WaterwayAxisImport) + return &WaterwayAxis{ + URL: wxi.URL, + FeatureType: wxi.FeatureType, + SortBy: nilString(wxi.SortBy), + } + }, + + WAJobKind: func(input interface{}) interface{} { + wai := input.(*models.WaterwayAreaImport) + return &WaterwayArea{ + URL: wai.URL, + FeatureType: wai.FeatureType, + SortBy: nilString(wai.SortBy), + } + }, + + WGJobKind: func(input interface{}) interface{} { + wgi := input.(*models.WaterwayGaugeImport) + return &WaterwayGauge{ + URL: wgi.URL, + Username: nilString(wgi.User), + Password: nilString(wgi.Password), + Insecure: wgi.Insecure, + } + }, + + DMVJobKind: func(input interface{}) interface{} { + dmvi := input.(*models.DistanceMarksVirtualImport) + return &DistanceMarksVirtual{ + URL: dmvi.URL, + Username: nilString(dmvi.User), + Password: nilString(dmvi.Password), + Insecure: dmvi.Insecure, + } + }, + + FDJobKind: func(input interface{}) interface{} { + fdi := input.(*models.FairwayDimensionImport) + return &FairwayDimension{ + URL: fdi.URL, + FeatureType: fdi.FeatureType, + SortBy: nilString(fdi.SortBy), + LOS: fdi.LOS, + MinWidth: fdi.MinWidth, + MaxWidth: fdi.MaxWidth, + Depth: fdi.Depth, + SourceOrganization: fdi.SourceOrganization, + } + }, + + DMAJobKind: func(input interface{}) interface{} { + dmai := input.(*models.DistanceMarksAshoreImport) + return &DistanceMarksAshore{ + URL: dmai.URL, + FeatureType: dmai.FeatureType, + SortBy: nilString(dmai.SortBy), + } + }, + + STJobKind: func(input interface{}) interface{} { + sti := input.(*models.StretchImport) + return &Stretch{ + Name: sti.Name, + From: sti.From, + To: sti.To, + ObjNam: sti.ObjNam, + NObjNam: sti.NObjNam, + Source: sti.Source, + Date: sti.Date, + Countries: sti.Countries, + } + }, +} + +func nilString(s *string) string { + if s != nil { + return *s + } + return "" +} + +func ConvertToInternal(kind JobKind, src interface{}) interface{} { + fn := convertModel[kind] + if fn == nil { + return nil + } + return fn(src) +} diff -r b4ba751e70a1 -r 079a1d35e076 pkg/imports/scheduled.go --- a/pkg/imports/scheduled.go Tue Jan 29 12:36:45 2019 +0100 +++ b/pkg/imports/scheduled.go Wed Jan 30 10:03:20 2019 +0100 @@ -16,165 +16,15 @@ import ( "context" "database/sql" - "errors" "fmt" "log" "time" "gemma.intevation.de/gemma/pkg/common" + "gemma.intevation.de/gemma/pkg/models" "gemma.intevation.de/gemma/pkg/scheduler" ) -// JobKindSetups maps JobKinds to special setup functions. -var JobKindSetups = map[JobKind]func(*IDConfig) (interface{}, error){ - - GMJobKind: func(cfg *IDConfig) (interface{}, error) { - log.Println("info: schedule 'gm' import") - insecure := cfg.Attributes.Bool("insecure") - return &GaugeMeasurement{ - URL: *cfg.URL, - Insecure: insecure, - }, nil - }, - - FAJobKind: func(cfg *IDConfig) (interface{}, error) { - log.Println("info: schedule 'fa' import") - insecure := cfg.Attributes.Bool("insecure") - return &FairwayAvailability{ - URL: *cfg.URL, - Insecure: insecure, - }, nil - }, - - BNJobKind: func(cfg *IDConfig) (interface{}, error) { - log.Println("info: schedule 'bn' import") - insecure := cfg.Attributes.Bool("insecure") - return &Bottleneck{ - URL: *cfg.URL, - Insecure: insecure, - }, nil - }, - - WXJobKind: func(cfg *IDConfig) (interface{}, error) { - log.Println("info: schedule 'wx' import") - ft, found := cfg.Attributes.Get("feature-type") - if !found { - return nil, errors.New("cannot find 'feature-type' attribute") - } - sb, found := cfg.Attributes.Get("sort-by") - if !found { - return nil, errors.New("cannot find 'sort-by' attribute") - } - return &WaterwayAxis{ - URL: *cfg.URL, - FeatureType: ft, - SortBy: sb, - }, nil - }, - - WAJobKind: func(cfg *IDConfig) (interface{}, error) { - log.Println("info: schedule 'wa' import") - ft, found := cfg.Attributes.Get("feature-type") - if !found { - return nil, errors.New("cannot find 'feature-type' attribute") - } - sb, found := cfg.Attributes.Get("sort-by") - if !found { - return nil, errors.New("cannot find 'sort-by' attribute") - } - return &WaterwayArea{ - URL: *cfg.URL, - FeatureType: ft, - SortBy: sb, - }, nil - }, - - FDJobKind: func(cfg *IDConfig) (interface{}, error) { - log.Println("info: schedule 'fd' import") - ft, found := cfg.Attributes.Get("feature-type") - if !found { - return nil, errors.New("cannot find 'feature-type' attribute") - } - sb, found := cfg.Attributes.Get("sort-by") - if !found { - return nil, errors.New("cannot find 'sort-by' attribute") - } - los, found := cfg.Attributes.Int("los") - if !found { - return nil, errors.New("cannot find 'los' attribute") - } - minWidth, found := cfg.Attributes.Int("min-width") - if !found { - return nil, errors.New("cannot find 'min-width' attribute") - } - maxWidth, found := cfg.Attributes.Int("max-width") - if !found { - return nil, errors.New("cannot find 'max-width' attribute") - } - depth, found := cfg.Attributes.Int("depth") - if !found { - return nil, errors.New("cannot find 'depth' attribute") - } - sourceOrganization, found := cfg.Attributes.Get("source-organization") - if !found { - return nil, errors.New("cannot find 'source-organization' attribute") - } - - return &FairwayDimension{ - URL: *cfg.URL, - FeatureType: ft, - SortBy: sb, - LOS: los, - MinWidth: minWidth, - MaxWidth: maxWidth, - Depth: depth, - SourceOrganization: sourceOrganization, - }, nil - }, - - WGJobKind: func(cfg *IDConfig) (interface{}, error) { - log.Println("info: schedule 'wg' import") - username, _ := cfg.Attributes.Get("username") - password, _ := cfg.Attributes.Get("password") - insecure := cfg.Attributes.Bool("insecure") - return &WaterwayGauge{ - URL: *cfg.URL, - Username: username, - Password: password, - Insecure: insecure, - }, nil - }, - - DMVJobKind: func(cfg *IDConfig) (interface{}, error) { - log.Println("info: schedule 'dvm' import") - username, _ := cfg.Attributes.Get("username") - password, _ := cfg.Attributes.Get("password") - insecure := cfg.Attributes.Bool("insecure") - return &DistanceMarksVirtual{ - URL: *cfg.URL, - Username: username, - Password: password, - Insecure: insecure, - }, nil - }, - DMAJobKind: func(cfg *IDConfig) (interface{}, error) { - log.Println("info: schedule 'dma' import") - ft, found := cfg.Attributes.Get("feature-type") - if !found { - return nil, errors.New("cannot find 'feature-type' attribute") - } - sb, found := cfg.Attributes.Get("sort-by") - if !found { - return nil, errors.New("cannot find 'sort-by' attribute") - } - return &DistanceMarksAshore{ - URL: *cfg.URL, - FeatureType: ft, - SortBy: sb, - }, nil - }, -} - func init() { run := func(cfgID int64) { jobID, err := RunConfiguredImport(cfgID) @@ -185,74 +35,87 @@ log.Printf("info: added import #%d to queue\n", jobID) } - for kind := range JobKindSetups { + for kind := range kindToImportModel { scheduler.RegisterAction(string(kind), run) } } // RunConfiguredImportContext runs an import configured from the database. func RunConfiguredImportContext(ctx context.Context, conn *sql.Conn, id int64) (int64, error) { - cfg, err := LoadIDConfigContext(ctx, conn, id) + cfg, err := LoadPersistentConfigContext(ctx, conn, id) return runConfiguredImport(id, cfg, err) } // RunConfiguredImport runs an import configured from the database. func RunConfiguredImport(id int64) (int64, error) { - cfg, err := loadIDConfig(id) + cfg, err := loadPersistentConfig(id) return runConfiguredImport(id, cfg, err) } -func runConfiguredImport(id int64, cfg *IDConfig, err error) (int64, error) { +func runConfiguredImport(id int64, cfg *PersistentConfig, err error) (int64, error) { if err != nil { return 0, err } if cfg == nil { - return 0, fmt.Errorf("no config found for id %d.\n", id) - } - if cfg.URL == nil { - return 0, errors.New("error: No URL specified") + return 0, fmt.Errorf("no config found for id %d.", id) } kind := JobKind(cfg.Kind) - setup := JobKindSetups[kind] - if setup == nil { - return 0, fmt.Errorf("unknown job kind: %s", cfg.Kind) + ctor := ImportModelForJobKind(kind) + if ctor == nil { + return 0, fmt.Errorf("no constructor for kind '%s'.", cfg.Kind) } - what, err := setup(cfg) - if err != nil { + what := ctor() + + // Fill the data structure + if err := cfg.Attributes.Unmarshal(what); err != nil { return 0, err } + converted := ConvertToInternal(kind, what) + if converted == nil { + return 0, fmt.Errorf("Conversion of model for kind '%s' failed.", kind) + } + var serialized string - if serialized, err = common.ToJSONString(what); err != nil { + if serialized, err = common.ToJSONString(converted); err != nil { return 0, err } - due, _ := cfg.Attributes.Time("due") + // Extract the job runtime parameters. + var ( + email bool + due time.Time + trys *int + waitRetry *time.Duration + ) - ret, found := cfg.Attributes.Int("retries") - var retries *int - if found { - retries = &ret + if gqc, ok := what.(models.QueueConfigurationGetter); ok { + qc := gqc.GetQueueConfiguration() + if qc.Due != nil { + due = qc.Due.Time + } + trys = qc.Trys + if qc.WaitRetry != nil { + waitRetry = &qc.WaitRetry.Duration + } } - dur, found := cfg.Attributes.Duration("wait-retry") - var waitRetry *time.Duration - if found { - waitRetry = &dur + if ge, ok := what.(models.EmailTypeGetter); ok { + email = ge.GetEmailType().Email } var jobID int64 if jobID, err = AddJob( kind, due, - retries, + trys, waitRetry, cfg.User, - cfg.SendEMail, + email, serialized, ); err != nil { return 0, err diff -r b4ba751e70a1 -r 079a1d35e076 pkg/models/bn.go --- a/pkg/models/bn.go Tue Jan 29 12:36:45 2019 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,28 +0,0 @@ -// This is Free Software under GNU Affero General Public License v >= 3.0 -// without warranty, see README.md and license for details. -// -// SPDX-License-Identifier: AGPL-3.0-or-later -// License-Filename: LICENSES/AGPL-3.0.txt -// -// Copyright (C) 2018 by via donau -// – Österreichische Wasserstraßen-Gesellschaft mbH -// Software engineering by Intevation GmbH -// -// Author(s): -// * Sascha L. Teichmann - -package models - -import "gemma.intevation.de/gemma/pkg/common" - -type BottleneckImport struct { - URL string `json:"url"` - Insecure bool `json:"insecure"` - SendEmail bool `json:"send-email"` - Attributes common.Attributes `json:"attributes,omitempty"` -} - -type Bottleneck struct { - ID string - ResponsibleCountry string -} diff -r b4ba751e70a1 -r 079a1d35e076 pkg/models/common.go --- a/pkg/models/common.go Tue Jan 29 12:36:45 2019 +0100 +++ b/pkg/models/common.go Wed Jan 30 10:03:20 2019 +0100 @@ -20,6 +20,8 @@ "fmt" "strings" "time" + + "gemma.intevation.de/gemma/pkg/common" ) var ( @@ -30,8 +32,6 @@ // WGS84 is the EPSG of the World Geodetic System 1984. const WGS84 = 4326 -const DateFormat = "2006-01-02" - type ( Date struct{ time.Time } // Country is a valid country 2 letter code. @@ -41,7 +41,7 @@ ) func (srd Date) MarshalJSON() ([]byte, error) { - return json.Marshal(srd.Format(DateFormat)) + return json.Marshal(srd.Format(common.DateFormat)) } func (srd *Date) UnmarshalJSON(data []byte) error { @@ -49,7 +49,7 @@ if err := json.Unmarshal(data, &s); err != nil { return err } - d, err := time.Parse(DateFormat, s) + d, err := time.Parse(common.DateFormat, s) if err == nil { *srd = Date{d} } diff -r b4ba751e70a1 -r 079a1d35e076 pkg/models/distancemarks.go --- a/pkg/models/distancemarks.go Tue Jan 29 12:36:45 2019 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,29 +0,0 @@ -// This is Free Software under GNU Affero General Public License v >= 3.0 -// without warranty, see README.md and license for details. -// -// SPDX-License-Identifier: AGPL-3.0-or-later -// License-Filename: LICENSES/AGPL-3.0.txt -// -// Copyright (C) 2018 by via donau -// – Österreichische Wasserstraßen-Gesellschaft mbH -// Software engineering by Intevation GmbH -// -// Author(s): -// * Sascha L. Teichmann - -package models - -import "gemma.intevation.de/gemma/pkg/common" - -type ( - // DistanceMarksVirtualImport specifies an import of distance marks virtual. - DistanceMarksVirtualImport struct { - // URL is the SOAP service URL. - URL string `json:"url"` - // SendEmail is set to true if an email should be send after - // importing the waterway gauges. - SendEmail bool `json:"send-email"` - // Attributes are optional attributes. - Attributes common.Attributes `json:"attributes,omitempty"` - } -) diff -r b4ba751e70a1 -r 079a1d35e076 pkg/models/dma.go --- a/pkg/models/dma.go Tue Jan 29 12:36:45 2019 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,33 +0,0 @@ -// This is Free Software under GNU Affero General Public License v >= 3.0 -// without warranty, see README.md and license for details. -// -// SPDX-License-Identifier: AGPL-3.0-or-later -// License-Filename: LICENSES/AGPL-3.0.txt -// -// Copyright (C) 2018 by via donau -// – Österreichische Wasserstraßen-Gesellschaft mbH -// Software engineering by Intevation GmbH -// -// Author(s): -// * Raimund Renkert - -package models - -import "gemma.intevation.de/gemma/pkg/common" - -type ( - // DistanceMarksAshoreImport specifies an import of the distance marks. - DistanceMarksAshoreImport struct { - // URL is the capabilities URL of the WFS. - URL string `json:"url"` - // FeatureType is the layer to use. - FeatureType string `json:"feature-type"` - // SortBy sorts the feature by this key. - SortBy string `json:"sort-by"` - // SendEmail is set to true if an email should be send after - // importing the axis. - SendEmail bool `json:"send-email"` - // Attributes are optional attributes. - Attributes common.Attributes `json:"attributes,omitempty"` - } -) diff -r b4ba751e70a1 -r 079a1d35e076 pkg/models/fa.go --- a/pkg/models/fa.go Tue Jan 29 12:36:45 2019 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,34 +0,0 @@ -// This is Free Software under GNU Affero General Public License v >= 3.0 -// without warranty, see README.md and license for details. -// -// SPDX-License-Identifier: AGPL-3.0-or-later -// License-Filename: LICENSES/AGPL-3.0.txt -// -// Copyright (C) 2018 by via donau -// – Österreichische Wasserstraßen-Gesellschaft mbH -// Software engineering by Intevation GmbH -// -// Author(s): -// * Raimund Renkert - -package models - -import ( - "time" - - "gemma.intevation.de/gemma/pkg/common" -) - -// FairwayAvailabilityImport contains data used to define the endpoint -type FairwayAvailabilityImport struct { - URL string `json:"url"` - Insecure bool `json:"insecure"` - SendEmail bool `json:"send-email"` - // Attributes are optional attributes. - Attributes common.Attributes `json:"attributes,omitempty"` -} - -type UniqueFairwayAvailability struct { - BottleneckId string - Surdat time.Time -} diff -r b4ba751e70a1 -r 079a1d35e076 pkg/models/fd.go --- a/pkg/models/fd.go Tue Jan 29 12:36:45 2019 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,43 +0,0 @@ -// This is Free Software under GNU Affero General Public License v >= 3.0 -// without warranty, see README.md and license for details. -// -// SPDX-License-Identifier: AGPL-3.0-or-later -// License-Filename: LICENSES/AGPL-3.0.txt -// -// Copyright (C) 2018 by via donau -// – Österreichische Wasserstraßen-Gesellschaft mbH -// Software engineering by Intevation GmbH -// -// Author(s): -// * Raimund Renkert - -package models - -import "gemma.intevation.de/gemma/pkg/common" - -type ( - // FairwayDimensionImport specifies an import of the waterway axis. - FairwayDimensionImport struct { - // URL is the capabilities URL of the WFS. - URL string `json:"url"` - // FeatureType is the layer to use. - FeatureType string `json:"feature-type"` - // SortBy sorts the feature by this key. - SortBy string `json:"sort-by"` - // SendEmail is set to true if an email should be send after - // importing the axis. - SendEmail bool `json:"send-email"` - // LOS is the level of service provided by the wfs - LOS int `json:"los"` - // MinWidth is the minimum width of the fairway for the specified LOS - MinWidth int `json:"min-width"` - // MaxWidth is the maximum width of the fairway for the specified LOS - MaxWidth int `json:"max-width"` - // Depth is the minimum depth of the fairway for the specified LOS - Depth int `json:"depth"` - // SourceOrganization specifies the source of the entry - SourceOrganization string `json:"source-organization"` - // Attributes are optional attributes. - Attributes common.Attributes `json:"attributes,omitempty"` - } -) diff -r b4ba751e70a1 -r 079a1d35e076 pkg/models/gauge.go --- a/pkg/models/gauge.go Tue Jan 29 12:36:45 2019 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,35 +0,0 @@ -// This is Free Software under GNU Affero General Public License v >= 3.0 -// without warranty, see README.md and license for details. -// -// SPDX-License-Identifier: AGPL-3.0-or-later -// License-Filename: LICENSES/AGPL-3.0.txt -// -// Copyright (C) 2018 by via donau -// – Österreichische Wasserstraßen-Gesellschaft mbH -// Software engineering by Intevation GmbH -// -// Author(s): -// * Raimund Renkert - -package models - -import ( - "time" - - "gemma.intevation.de/gemma/pkg/common" -) - -// GaugeMeasurementImport contains data used to define the endpoint -type GaugeMeasurementImport struct { - URL string `json:"url"` - Insecure bool `json:"insecure"` - SendEmail bool `json:"send-email"` - // Attributes are optional attributes. - Attributes common.Attributes `json:"attributes,omitempty"` -} - -// GaugeMeasurement holds information about a gauge and the latest measurement -type GaugeMeasurement struct { - Gauge Isrs - LatestDateIssue time.Time -} diff -r b4ba751e70a1 -r 079a1d35e076 pkg/models/importbase.go --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pkg/models/importbase.go Wed Jan 30 10:03:20 2019 +0100 @@ -0,0 +1,184 @@ +// This is Free Software under GNU Affero General Public License v >= 3.0 +// without warranty, see README.md and license for details. +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// License-Filename: LICENSES/AGPL-3.0.txt +// +// Copyright (C) 2018 by via donau +// – Österreichische Wasserstraßen-Gesellschaft mbH +// Software engineering by Intevation GmbH +// +// Author(s): +// * Sascha L. Teichmann + +package models + +import ( + "encoding/json" + "errors" + "time" + + "gemma.intevation.de/gemma/pkg/common" + "github.com/robfig/cron" +) + +type ( + // CronSpec is a string containing a cron line. + CronSpec string + + ConfigTime struct{ time.Time } + + ConfigDuration struct{ time.Duration } + + EmailType struct { + Email bool `json:"send-email,omitempty"` + } + + QueueConfigurationType struct { + EmailType + Trys *int `json:"trys,omitempty"` + WaitRetry *ConfigDuration `json:"wait-retry,omitempty"` + Due *ConfigTime `json:"due,omitempty"` + Cron *CronSpec `json:"cron,omitempty"` + } + + URLType struct { + URL string `json:"url"` + Insecure bool `json:"insecure,omitempty"` + User *string `json:"user,omitempty"` + Password *string `json:"password,omitempty"` + } + + EmailTypeGetter interface { + GetEmailType() *EmailType + } + + QueueConfigurationGetter interface { + GetQueueConfiguration() *QueueConfigurationType + } + + URLTypeGetter interface { + GetURLType() *URLType + } +) + +func (qct *QueueConfigurationType) GetQueueConfiguration() *QueueConfigurationType { + return qct +} + +func (ut *URLType) GetURLType() *URLType { + return ut +} + +func (et *EmailType) GetEmailType() *EmailType { + return et +} + +func (cd *ConfigDuration) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + dur, err := time.ParseDuration(s) + if err != nil { + return err + } + if dur < 0 { + return errors.New("duration has to be none negative.") + } + *cd = ConfigDuration{dur} + return nil +} + +func (cd *ConfigDuration) MarshalJSON() ([]byte, error) { + s := cd.Duration.String() + return json.Marshal([]byte(s)) +} + +func (ct *ConfigTime) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + t, err := time.Parse(common.TimeFormat, s) + if err != nil { + return err + } + *ct = ConfigTime{t} + return nil +} + +func (ct *ConfigTime) MarshalJSON() ([]byte, error) { + s := ct.Time.Format(common.TimeFormat) + return json.Marshal([]byte(s)) +} + +// UnmarshalJSON checks if the incoming string is a valid cron line. +func (cs *CronSpec) UnmarshalJSON(data []byte) error { + var spec string + if err := json.Unmarshal(data, &spec); err != nil { + return err + } + if _, err := cron.Parse(spec); err != nil { + return err + } + *cs = CronSpec(spec) + return nil +} + +func (et *EmailType) MarshalAttributes(attrs common.Attributes) error { + if et.Email { + attrs.SetBool("email", et.Email) + } + return nil +} + +func (et *EmailType) UnmarshalAttributes(attrs common.Attributes) error { + et.Email = attrs.Bool("email") + return nil +} + +func (qct *QueueConfigurationType) MarshalAttributes(attrs common.Attributes) error { + if err := qct.EmailType.MarshalAttributes(attrs); err != nil { + return err + } + if qct.Trys != nil { + attrs.SetInt("trys", *qct.Trys) + } + if qct.WaitRetry != nil { + attrs.SetDuration("wait-retry", qct.WaitRetry.Duration) + } + if qct.Due != nil { + attrs.SetTime("due", qct.Due.Time) + } + return nil +} + +func (ut *URLType) MarshalAttributes(attrs common.Attributes) error { + attrs.Set("url", ut.URL) + if ut.Insecure { + attrs.SetBool("insecure", ut.Insecure) + } + if ut.User != nil { + attrs.Set("user", *ut.User) + } + if ut.Password != nil { + attrs.Set("password", *ut.Password) + } + return nil +} + +func (ut *URLType) UnmarshalAttributes(attrs common.Attributes) error { + url, found := attrs.Get("url") + if !found { + return errors.New("missing 'url' attribute") + } + ut.URL = url + if user, found := attrs.Get("user"); found { + ut.User = &user + } + if password, found := attrs.Get("password"); found { + ut.Password = &password + } + return nil +} diff -r b4ba751e70a1 -r 079a1d35e076 pkg/models/imports.go --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pkg/models/imports.go Wed Jan 30 10:03:20 2019 +0100 @@ -0,0 +1,264 @@ +// This is Free Software under GNU Affero General Public License v >= 3.0 +// without warranty, see README.md and license for details. +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// License-Filename: LICENSES/AGPL-3.0.txt +// +// Copyright (C) 2018 by via donau +// – Österreichische Wasserstraßen-Gesellschaft mbH +// Software engineering by Intevation GmbH +// +// Author(s): +// * Sascha L. Teichmann +package models + +import ( + "errors" + "strings" + + "gemma.intevation.de/gemma/pkg/common" +) + +type ( + ConfigurableURLImport struct { + URLType + QueueConfigurationType + } + + BottleneckImport struct { + ConfigurableURLImport + } + + // GaugeMeasurementImport contains data used to define the endpoint + GaugeMeasurementImport struct { + ConfigurableURLImport + } + + // FairwayAvailabilityImport contains data used to define the endpoint + FairwayAvailabilityImport struct { + ConfigurableURLImport + } + + // WaterwayAxisImport specifies an import of waterway gauges. + WaterwayGaugeImport struct { + ConfigurableURLImport + } + + // DistanceMarksVirtualImport specifies an import of distance marks virtual. + DistanceMarksVirtualImport struct { + ConfigurableURLImport + } + + WFSImport struct { + ConfigurableURLImport + + // FeatureType is the layer to use. + FeatureType string `json:"feature-type"` + // SortBy sorts the feature by this key. + SortBy *string `json:"sort-by"` + } + + // WaterwayAxisImport specifies an import of the waterway axis. + WaterwayAxisImport struct { + WFSImport + } + + // WaterwayAreaImport specifies an import of the waterway area. + WaterwayAreaImport struct { + WFSImport + } + + // DistanceMarksAshoreImport specifies an import of the distance marks. + DistanceMarksAshoreImport struct { + WFSImport + } + + // FairwayDimensionImport specifies an import of the waterway axis. + FairwayDimensionImport struct { + WFSImport + + // LOS is the level of service provided by the wfs + LOS int `json:"los"` + // MinWidth is the minimum width of the fairway for the specified LOS + MinWidth int `json:"min-width"` + // MaxWidth is the maximum width of the fairway for the specified LOS + MaxWidth int `json:"max-width"` + // Depth is the minimum depth of the fairway for the specified LOS + Depth int `json:"depth"` + // SourceOrganization specifies the source of the entry + SourceOrganization string `json:"source-organization"` + } + + StretchImport struct { + EmailType + + Name string `json:"name"` + From Isrs `json:"from"` + To Isrs `json:"to"` + ObjNam string `json:"objnam"` + NObjNam *string `json:"nobjnam"` + Source string `json:"source-organization"` + Date Date `json:"date-info"` + Countries UniqueCountries `json:"countries"` + } +) + +func (cui *ConfigurableURLImport) MarshalAttributes(attrs common.Attributes) error { + return cui.URLType.MarshalAttributes(attrs) +} + +func (cui *ConfigurableURLImport) UnmarshalAttributes(attrs common.Attributes) error { + return cui.URLType.UnmarshalAttributes(attrs) +} + +func (wi *WFSImport) MarshalAttributes(attrs common.Attributes) error { + if err := wi.ConfigurableURLImport.MarshalAttributes(attrs); err != nil { + return err + } + attrs.Set("feature-type", wi.FeatureType) + if wi.SortBy != nil { + attrs.Set("sort-by", *wi.SortBy) + } + return nil +} + +func (wi *WFSImport) UnmarshalAttributes(attrs common.Attributes) error { + if err := wi.URLType.UnmarshalAttributes(attrs); err != nil { + return err + } + ft, found := attrs.Get("feature-type") + if !found { + return errors.New("missing 'feature-type' attribute") + } + wi.FeatureType = ft + if sb, found := attrs.Get("sort-by"); found { + wi.SortBy = &sb + } + return nil +} + +func (fdi *FairwayDimensionImport) MarshalAttributes(attrs common.Attributes) error { + if err := fdi.WFSImport.MarshalAttributes(attrs); err != nil { + return err + } + attrs.SetInt("los", fdi.LOS) + attrs.SetInt("min-width", fdi.MinWidth) + attrs.SetInt("max-width", fdi.MaxWidth) + attrs.SetInt("depth", fdi.Depth) + attrs.Set("source-organization", fdi.SourceOrganization) + return nil +} + +func (fdi *FairwayDimensionImport) UnmarshalAttributes(attrs common.Attributes) error { + if err := fdi.WFSImport.UnmarshalAttributes(attrs); err != nil { + return err + } + los, found := attrs.Int("los") + if !found { + return errors.New("missing 'los' attribute") + } + fdi.LOS = los + minWidth, found := attrs.Int("min-width") + if !found { + return errors.New("missing 'min-width' attribute") + } + fdi.MinWidth = minWidth + maxWidth, found := attrs.Int("max-width") + if !found { + return errors.New("missing 'max-width' attribute") + } + fdi.MaxWidth = maxWidth + depth, found := attrs.Int("depth") + if !found { + return errors.New("missing 'depth' attribute") + } + fdi.Depth = depth + source, found := attrs.Get("source-organization") + if !found { + return errors.New("missing 'source-organization' attribute") + } + fdi.SourceOrganization = source + return nil +} + +func (sti *StretchImport) MarshalAttributes(attrs common.Attributes) error { + if err := sti.EmailType.MarshalAttributes(attrs); err != nil { + return err + } + attrs.Set("name", sti.Name) + attrs.Set("from", sti.From.String()) + attrs.Set("to", sti.To.String()) + attrs.Set("objnam", sti.ObjNam) + if sti.NObjNam != nil { + attrs.Set("nobjnam", *sti.NObjNam) + } + attrs.Set("source-organization", sti.Source) + attrs.SetDate("date-info", sti.Date.Time) + if len(sti.Countries) > 0 { + countries := make([]string, len(sti.Countries)) + for i, c := range sti.Countries { + countries[i] = string(c) + } + attrs.Set("countries", strings.Join(countries, ",")) + } + + return nil +} + +func (sti *StretchImport) UnmarshalAttributes(attrs common.Attributes) error { + if err := sti.EmailType.UnmarshalAttributes(attrs); err != nil { + return err + } + name, found := attrs.Get("name") + if !found { + return errors.New("missing 'name' attribute") + } + sti.Name = name + from, found := attrs.Get("from") + if !found { + return errors.New("missing 'from' attribute") + } + f, err := IsrsFromString(from) + if err != nil { + return err + } + sti.From = *f + to, found := attrs.Get("from") + if !found { + return errors.New("missing 'to' attribute") + } + t, err := IsrsFromString(to) + if err != nil { + return err + } + sti.To = *t + objnam, found := attrs.Get("objnam") + if !found { + return errors.New("missing 'objnam' attribute") + } + sti.ObjNam = objnam + nobjnam, found := attrs.Get("nobjnam") + if found { + sti.NObjNam = &nobjnam + } + source, found := attrs.Get("source-organization") + if !found { + return errors.New("missing 'source' attribute") + } + sti.Source = source + date, found := attrs.Date("date-info") + if !found { + return errors.New("missing 'date-info' attribute") + } + sti.Date = Date{date} + countries, found := attrs.Get("countries") + if found { + csp := strings.Split(countries, ",") + cs := make(UniqueCountries, len(csp)) + for i, c := range csp { + cs[i] = Country(c) + } + sti.Countries = cs + } + return nil +} diff -r b4ba751e70a1 -r 079a1d35e076 pkg/models/stretch.go --- a/pkg/models/stretch.go Tue Jan 29 12:36:45 2019 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,30 +0,0 @@ -// This is Free Software under GNU Affero General Public License v >= 3.0 -// without warranty, see README.md and license for details. -// -// SPDX-License-Identifier: AGPL-3.0-or-later -// License-Filename: LICENSES/AGPL-3.0.txt -// -// Copyright (C) 2018 by via donau -// – Österreichische Wasserstraßen-Gesellschaft mbH -// Software engineering by Intevation GmbH -// -// Author(s): -// * Sascha L. Teichmann - -package models - -import "gemma.intevation.de/gemma/pkg/common" - -type StretchImport struct { - Name string `json:"name"` - From Isrs `json:"from"` - To Isrs `json:"to"` - ObjNam string `json:"objnam"` - NObjNam *string `json:"nobjnam"` - Source string `json:"source-organization"` - Date Date `json:"date-info"` - Countries UniqueCountries `json:"countries"` - - SendEmail bool `json:"send-email"` - Attributes common.Attributes `json:"attributes,omitempty"` -} diff -r b4ba751e70a1 -r 079a1d35e076 pkg/models/waterway.go --- a/pkg/models/waterway.go Tue Jan 29 12:36:45 2019 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,59 +0,0 @@ -// This is Free Software under GNU Affero General Public License v >= 3.0 -// without warranty, see README.md and license for details. -// -// SPDX-License-Identifier: AGPL-3.0-or-later -// License-Filename: LICENSES/AGPL-3.0.txt -// -// Copyright (C) 2018 by via donau -// – Österreichische Wasserstraßen-Gesellschaft mbH -// Software engineering by Intevation GmbH -// -// Author(s): -// * Sascha L. Teichmann - -package models - -import "gemma.intevation.de/gemma/pkg/common" - -type ( - // WaterwayAxisImport specifies an import of the waterway axis. - WaterwayAxisImport struct { - // URL is the capabilities URL of the WFS. - URL string `json:"url"` - // FeatureType is the layer to use. - FeatureType string `json:"feature-type"` - // SortBy sorts the feature by this key. - SortBy string `json:"sort-by"` - // SendEmail is set to true if an email should be send after - // importing the axis. - SendEmail bool `json:"send-email"` - // Attributes are optional attributes. - Attributes common.Attributes `json:"attributes,omitempty"` - } - - // WaterwayAreaImport specifies an import of the waterway area. - WaterwayAreaImport struct { - // URL is the capabilities URL of the WFS. - URL string `json:"url"` - // FeatureType is the layer to use. - FeatureType string `json:"feature-type"` - // SortBy sorts the feature by this key. - SortBy string `json:"sort-by"` - // SendEmail is set to true if an email should be send after - // importing the axis. - SendEmail bool `json:"send-email"` - // Attributes are optional attributes. - Attributes common.Attributes `json:"attributes,omitempty"` - } - - // WaterwayAxisImport specifies an import of waterway gauges. - WaterwayGaugeImport struct { - // URL is the SOAP service URL. - URL string `json:"url"` - // SendEmail is set to true if an email should be send after - // importing the waterway gauges. - SendEmail bool `json:"send-email"` - // Attributes are optional attributes. - Attributes common.Attributes `json:"attributes,omitempty"` - } -) diff -r b4ba751e70a1 -r 079a1d35e076 pkg/scheduler/boot.go --- a/pkg/scheduler/boot.go Tue Jan 29 12:36:45 2019 +0100 +++ b/pkg/scheduler/boot.go Wed Jan 30 10:03:20 2019 +0100 @@ -26,13 +26,24 @@ bootRole = "sys_admin" selectImportConfSQL = ` -SELECT id, username, cron -FROM import.import_configuration -WHERE cron IS NOT NULL` +SELECT c.id, username, a.v +FROM import.import_configuration c +JOIN import.import_configuration_attributes a +ON c.id = a.import_configuration_id +WHERE c.id IN ( + SELECT import_configuration_id + FROM import.import_configuration_attributes + WHERE k = 'cron')` scheduledIDsSQL = ` -SELECT id from import.import_configuration -WHERE username = $1 AND cron IS NOT NULL` +SELECT id +FROM import.import_configuration +WHERE + username = $1 + AND id IN ( + SELECT import_configuration_id + FROM import.import_configuration_attributes + WHERE k = 'cron')` ) func init() { go boot() } diff -r b4ba751e70a1 -r 079a1d35e076 schema/gemma.sql --- a/schema/gemma.sql Tue Jan 29 12:36:45 2019 +0100 +++ b/schema/gemma.sql Wed Jan 30 10:03:20 2019 +0100 @@ -630,10 +630,7 @@ REFERENCES internal.user_profiles(username) ON DELETE CASCADE ON UPDATE CASCADE, - kind varchar NOT NULL, - send_email boolean NOT NULL DEFAULT false, - cron varchar, - url varchar + kind varchar NOT NULL ) CREATE TABLE import_configuration_attributes (