Mercurial > kallithea
view Jenkinsfile @ 8769:d35d14b05b82
diff: handle some escaped characters in Git diffs
There are some odd characters (like \r and \n) that the Kallithea UI doesn't
allow in filenames in repos. Kallithea (through the routes module) will fail to
generate URLs when browsing Files. That is a known limitation with minimal
real-world impact, non-trivial to work around or fix.
There are very few relevant use cases for tracking files with odd filenames. \t
is valid but is hard to render in a meaningful way in the UI. And ASCII
characters like \ and " are not usable on Windows and should just be avoided.
Kallithea would parse Git diffs with odd characers incorrectly or fail, even
before hitting the known limitation. With this change, Kallithea will parse
diffs with odd filenames correctly (and then hit the limitation).
Git will quote odd filenames and escape the odd characters when emitting diffs.
(Mercurial does by design not allow \r and \n , and Mercurial will thus never
have to quote file names in diffs.)
Quotes are already handled (and ignored). With this change, Kallithea will
handle \ unescaping of \\ and \", the usual letters like \r and \n and \t, and
octal numbers like \033 (for ESC) .
Filenames with \ and " will work perfectly (when not on Windows).
Filenames with \t and ESC will work fine, but without helpful display in the
UI.
Filenames with \r and \n will still make the UI fail when trying to generate
URLs.
Thanks to stypr of Flatt Security for raising this.
author | Mads Kiilerich <mads@kiilerich.com> |
---|---|
date | Sat, 14 Nov 2020 15:41:39 +0100 |
parents | 89e9aef9b983 |
children | 0a9ddb8cd8c1 |
line wrap: on
line source
def createvirtualenv = '' def activatevirtualenv = '' node { properties([[$class: 'BuildDiscarderProperty', strategy: [$class: 'LogRotator', artifactDaysToKeepStr: '', artifactNumToKeepStr: '10', daysToKeepStr: '', numToKeepStr: '']]]); if (isUnix()) { createvirtualenv = 'rm -r $JENKINS_HOME/venv/$JOB_NAME || true && python3 -m venv $JENKINS_HOME/venv/$JOB_NAME' activatevirtualenv = '. $JENKINS_HOME/venv/$JOB_NAME/bin/activate' } else { createvirtualenv = 'rmdir /s /q %JENKINS_HOME%\\venv\\%JOB_NAME% || true && python3 -m venv %JENKINS_HOME%\\venv\\%JOB_NAME%' activatevirtualenv = 'call %JENKINS_HOME%\\venv\\%JOB_NAME%\\Scripts\\activate.bat' } stage('checkout') { checkout scm if (isUnix()) { sh 'hg --config extensions.purge= purge --all' } else { bat 'hg --config extensions.purge= purge --all' } } stage('virtual env') { def virtualenvscript = """$createvirtualenv $activatevirtualenv python -m pip install --upgrade pip pip install --upgrade setuptools pip install --upgrade pylint pip install --upgrade pytest-cov """ if (isUnix()) { virtualenvscript += """ pip install --upgrade python-ldap pip install --upgrade python-pam """ sh virtualenvscript } else { bat virtualenvscript } } stage('setup') { def virtualenvscript = """$activatevirtualenv pip install --upgrade -e . -r dev_requirements.txt python setup.py compile_catalog """ if (isUnix()) { sh virtualenvscript } else { bat virtualenvscript } stash name: 'kallithea', useDefaultExcludes: false } stage('pylint') { sh script: """$activatevirtualenv pylint -j 0 --disable=C -f parseable kallithea > pylint.out """, returnStatus: true archiveArtifacts 'pylint.out' try { step([$class: 'WarningsPublisher', canComputeNew: false, canResolveRelativePaths: false, defaultEncoding: '', excludePattern: '', healthy: '', includePattern: '', messagesPattern: '', parserConfigurations: [[parserName: 'PyLint', pattern: 'pylint.out']], unHealthy: '']) } catch (java.lang.IllegalArgumentException exc) { echo "You need to install the 'Warnings Plug-in' to display the pylint report." currentBuild.result = 'UNSTABLE' echo "Caught: ${exc}" } } } def pytests = [:] pytests['sqlite'] = { node { ws { deleteDir() unstash name: 'kallithea' if (isUnix()) { sh script: """$activatevirtualenv py.test -p no:sugar --cov-config .coveragerc --junit-xml=pytest_sqlite.xml --cov=kallithea """, returnStatus: true } else { bat script: """$activatevirtualenv py.test -p no:sugar --cov-config .coveragerc --junit-xml=pytest_sqlite.xml --cov=kallithea """, returnStatus: true } sh 'sed --in-place "s/\\(classname=[\'\\"]\\)/\\1SQLITE./g" pytest_sqlite.xml' archiveArtifacts 'pytest_sqlite.xml' junit 'pytest_sqlite.xml' writeFile(file: '.coverage.sqlite', text: readFile('.coverage')) stash name: 'coverage.sqlite', includes: '.coverage.sqlite' } } } pytests['de'] = { node { if (isUnix()) { ws { deleteDir() unstash name: 'kallithea' withEnv(['LANG=de_DE.UTF-8', 'LANGUAGE=de', 'LC_ADDRESS=de_DE.UTF-8', 'LC_IDENTIFICATION=de_DE.UTF-8', 'LC_MEASUREMENT=de_DE.UTF-8', 'LC_MONETARY=de_DE.UTF-8', 'LC_NAME=de_DE.UTF-8', 'LC_NUMERIC=de_DE.UTF-8', 'LC_PAPER=de_DE.UTF-8', 'LC_TELEPHONE=de_DE.UTF-8', 'LC_TIME=de_DE.UTF-8', ]) { sh script: """$activatevirtualenv py.test -p no:sugar --cov-config .coveragerc --junit-xml=pytest_de.xml --cov=kallithea """, returnStatus: true } sh 'sed --in-place "s/\\(classname=[\'\\"]\\)/\\1DE./g" pytest_de.xml' archiveArtifacts 'pytest_de.xml' junit 'pytest_de.xml' writeFile(file: '.coverage.de', text: readFile('.coverage')) stash name: 'coverage.de', includes: '.coverage.de' } } } } pytests['mysql'] = { node { if (isUnix()) { ws { deleteDir() unstash name: 'kallithea' sh """$activatevirtualenv pip install --upgrade MySQL-python """ withEnv(['TEST_DB=mysql://kallithea:kallithea@jenkins_mysql/kallithea_test?charset=utf8']) { if (isUnix()) { sh script: """$activatevirtualenv py.test -p no:sugar --cov-config .coveragerc --junit-xml=pytest_mysql.xml --cov=kallithea """, returnStatus: true } else { bat script: """$activatevirtualenv py.test -p no:sugar --cov-config .coveragerc --junit-xml=pytest_mysql.xml --cov=kallithea """, returnStatus: true } } sh 'sed --in-place "s/\\(classname=[\'\\"]\\)/\\1MYSQL./g" pytest_mysql.xml' archiveArtifacts 'pytest_mysql.xml' junit 'pytest_mysql.xml' writeFile(file: '.coverage.mysql', text: readFile('.coverage')) stash name: 'coverage.mysql', includes: '.coverage.mysql' } } } } pytests['postgresql'] = { node { if (isUnix()) { ws { deleteDir() unstash name: 'kallithea' sh """$activatevirtualenv pip install --upgrade psycopg2 """ withEnv(['TEST_DB=postgresql://kallithea:kallithea@jenkins_postgresql/kallithea_test']) { if (isUnix()) { sh script: """$activatevirtualenv py.test -p no:sugar --cov-config .coveragerc --junit-xml=pytest_postgresql.xml --cov=kallithea """, returnStatus: true } else { bat script: """$activatevirtualenv py.test -p no:sugar --cov-config .coveragerc --junit-xml=pytest_postgresql.xml --cov=kallithea """, returnStatus: true } } sh 'sed --in-place "s/\\(classname=[\'\\"]\\)/\\1POSTGRES./g" pytest_postgresql.xml' archiveArtifacts 'pytest_postgresql.xml' junit 'pytest_postgresql.xml' writeFile(file: '.coverage.postgresql', text: readFile('.coverage')) stash name: 'coverage.postgresql', includes: '.coverage.postgresql' } } } } stage('Tests') { parallel pytests node { unstash 'coverage.sqlite' unstash 'coverage.de' unstash 'coverage.mysql' unstash 'coverage.postgresql' if (isUnix()) { sh script: """$activatevirtualenv coverage combine .coverage.sqlite .coverage.de .coverage.mysql .coverage.postgresql coverage xml """, returnStatus: true } else { bat script: """$activatevirtualenv coverage combine .coverage.sqlite .coverage.de .coverage.mysql .coverage.postgresql coverage xml """, returnStatus: true } try { step([$class: 'CoberturaPublisher', autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: 'coverage.xml', failNoReports: false, failUnhealthy: false, failUnstable: false, maxNumberOfBuilds: 0, onlyStable: false, zoomCoverageChart: false]) } catch (java.lang.IllegalArgumentException exc) { echo "You need to install the pipeline compatible 'CoberturaPublisher Plug-in' to display the coverage report." currentBuild.result = 'UNSTABLE' echo "Caught: ${exc}" } } }