Verified Commit 56353697 authored by Elias Häußler's avatar Elias Häußler 🐛

[FEATURE] Separate Gulp tasks in files and use updated Babel packages

parent 4e8cb3ee
{
"presets": ["@babel/env"]
}
......@@ -30,60 +30,58 @@ git clone https://github.com/eliashaeussler/student-growth.git
cd student-growth
```
After cloning install the Node dependencies:
After cloning install the dependencies:
```bash
yarn install
yarn
```
You can also use `npm install` in case you haven't installed `yarn`.
## Build the site
In order to access the website which shows the visualization, you need to build the necessary project files:
In order to access the website which shows the visualization you need to build the necessary project files:
```bash
yarn run build
yarn run serve
```
You can alternatively use `npm run-script build` or `gulp --production && gulp server`.
This compiles all necessary files and places them inside a `dist` folder. Files from this folder will be used as source
for a web server which automatically starts running when you run one of the above commands.
for a web server which automatically starts running when you run the above commands.
If your Browser does not open by it's own, open it manually and navigate to the following page:
<http://localhost:8000>
If port `8000` is already in use, BrowserSync tries to use the next available port. You can see the active port in your
command line after running one of the above commands.
CLI after running the above commands.
### Development mode
```bash
yarn run start
yarn run dev
```
(alternatively: `npm start` or simply `gulp`)
The Development mode allows you to see changes on project files immediately in your browser since BrowserSync is watching
for changes.
### Other tasks
### Available scripts
| `npm` script | Gulp task | Description |
| ---------- | --------- | ----------- |
| `sources` | `default --production` | Build only the source files and does not start the server |
| `serve` | `serve` | Run the server without creating any source files |
| `watch` | `server watch` | Run the server without creating source files and watch for changes |
| `test` | – | Test the project (does not start the server) |
`npm` script | Gulp task | Description
------------ | ----------------------- | -----------
`build` | `NODE_ENV=prod default` | Build the project files for productional use
`dev` | `default` | Build the dev project files and start server
`serve` | `serve` | Run the server without creating any source files
`test` | `lint` | Run linters to test project files
All `npm` scripts need to be executed using `yarn run <command>` or `npm run-script <command>`.
When running Gulp tasks, execute them using `gulp <command>`.
You can provide the parameter `--force-download` to download latest data before running the project. Alternatively,
it can be downloaded manually by running the Python script.
## Download the latest data
......@@ -106,20 +104,20 @@ Download successful: src/data/data.csv
The file `data/source.json` contains important information which are being used to parse the remote source file correctly.
| Key | Description | Configuration |
| --- | --- | --- |
| `url` | File which contains information about the source files and their file formats | Valid URL with protocol as string |
| `keys` | Position of keys (headlines) inside the source file (X and Y) | Object with keys `x` and `y` which contain arrays with numbers |
| `data_rows` | Position of first and last data row | Object with keys `first` and `last` as numbers. `last` can be a row number (e.g. `293`) or the difference from the last available row (e.g. `-4`, more flexible)
Key | Description | Configuration
----------- | ----------------------------------------------------------------------------- | -------------
`url` | File which contains information about the source files and their file formats | Valid URL with protocol as string
`keys` | Position of keys (headlines) inside the source file (X and Y) | Object with keys `x` and `y` which contain arrays with numbers
`data_rows` | Position of first and last data row | Object with keys `first` and `last` as numbers. `last` can be a row number (e.g. `293`) or the difference from the last available row (e.g. `-4`, more flexible)
### Script parameters
You can pass additional arguments when calling the download script. Run `python3 bin/download.py -h` to get an overview.
| Parameter | Description |
| --- | --- |
| `-q`, `--quiet` | Suppresses status messages (error messages will still be displayed) |
| `-u`, `--unsafe` | Skip integrity check of requested CSV file |
Parameter | Description
---------------- | -----------
`-q`, `--quiet` | Suppresses status messages (error messages will still be displayed)
`-u`, `--unsafe` | Skip integrity check of requested CSV file
### Installing certificates on macOS
......
COMPATIBILITY:
- "last 2 versions"
- "ie >= 8"
- "ios >= 7"
PATHS:
dist: "dist"
data: "src/data/**/*"
images:
- "src/assets/img/**/*"
javascript:
src: "src/assets/js/**/*.js"
entry: "src/assets/js/main.js"
sass:
src: "src/assets/scss/**/*.scss"
entry: "src/assets/scss/main.scss"
modules:
- "node_modules/reset-css"
- "node_modules/spinkit/scss"
pages: "src/**/*.html"
SERVER:
port: 8000
/*
* Copyright (c) 2019 Elias Häußler <elias@haeussler.dev>. All rights reserved.
*/
const yaml = require('js-yaml');
const fs = require('fs');
const config = yaml.safeLoad(fs.readFileSync(__dirname + '/../config.yml'), 'utf8');
module.exports = {
PRODUCTION: process.env.NODE_ENV === 'prod',
DOWNLOAD: process.argv.includes('--force-download'),
...config
};
/*
* Copyright (c) 2019 Elias Häußler <elias@haeussler.dev>. All rights reserved.
*/
const rimraf = require('rimraf');
const {PATHS} = require('../config');
module.exports = cb => rimraf(PATHS.dist, cb);
/*
* Copyright (c) 2019 Elias Häußler <elias@haeussler.dev>. All rights reserved.
*/
const {dest, src} = require('gulp');
const {PATHS} = require('../config');
module.exports = () => src(PATHS.data, { nodir: true }).pipe(dest(PATHS.dist + '/data'));
/*
* Copyright (c) 2019 Elias Häußler <elias@haeussler.dev>. All rights reserved.
*/
const {spawn} = require('child_process');
const {DOWNLOAD} = require('../config');
module.exports = cb => {
if (DOWNLOAD) {
const pythonProcess = spawn('python3', [__dirname + '/../../bin/download.py']);
pythonProcess.stdout.on('data', data => console.log(String(data)));
}
if ("function" === typeof cb) cb();
};
/*
* Copyright (c) 2019 Elias Häußler <elias@haeussler.dev>. All rights reserved.
*/
const {dest, src} = require('gulp');
const sourcemaps = require('gulp-sourcemaps');
const when = require('gulp-if');
const named = require('vinyl-named');
const webpackStream = require('webpack-stream');
const webpack = require('webpack');
const {PATHS, PRODUCTION} = require('../config');
module.exports = () => src(PATHS.javascript.entry, { nodir: true })
.pipe(named())
.pipe(sourcemaps.init())
.pipe(webpackStream(require('../../webpack.config'), webpack))
.pipe(when(!PRODUCTION, sourcemaps.write()))
.pipe(dest(PATHS.dist + '/assets/js'));
/*
* Copyright (c) 2019 Elias Häußler <elias@haeussler.dev>. All rights reserved.
*/
const {src} = require('gulp');
const jshint = require('gulp-jshint');
const {PATHS} = require('../config');
module.exports = () => src(PATHS.javascript.src)
.pipe(jshint())
.pipe(jshint.reporter('default'));
/*
* Copyright (c) 2019 Elias Häußler <elias@haeussler.dev>. All rights reserved.
*/
const {dest, src} = require('gulp');
const {PATHS} = require('../config');
module.exports = () => src(PATHS.pages).pipe(dest(PATHS.dist));
/*
* Copyright (c) 2019 Elias Häußler <elias@haeussler.dev>. All rights reserved.
*/
const browser = require('browser-sync');
module.exports = cb => {
browser.reload();
if ("function" === typeof cb) cb();
};
/*
* Copyright (c) 2019 Elias Häußler <elias@haeussler.dev>. All rights reserved.
*/
const {dest, src} = require('gulp');
const sourcemaps = require('gulp-sourcemaps');
const sass = require('gulp-sass');
const autoprefixer = require('gulp-autoprefixer');
const cleanCss = require('gulp-clean-css');
const named = require('vinyl-named');
const when = require('gulp-if');
const {COMPATIBILITY, PATHS, PRODUCTION} = require('../config');
module.exports = () => src(PATHS.sass.entry)
.pipe(named())
.pipe(sourcemaps.init())
.pipe(sass({
includePaths: [PATHS.sass.entry].concat(PATHS.sass.modules)
}).on('error', sass.logError))
.pipe(autoprefixer({browsers: COMPATIBILITY}))
.pipe(when(PRODUCTION, cleanCss({
compatibility: 'ie8',
level: {
1: {
specialComments: 1
}
}
})))
.pipe(when(!PRODUCTION, sourcemaps.write()))
.pipe(dest(PATHS.dist + '/assets/css'));
/*
* Copyright (c) 2019 Elias Häußler <elias@haeussler.dev>. All rights reserved.
*/
const {src} = require('gulp');
const sassLint = require('gulp-sass-lint');
const {PATHS} = require('../config');
module.exports = () => src(PATHS.sass.src)
.pipe(sassLint())
.pipe(sassLint.format())
.pipe(sassLint.failOnError());
/*
* Copyright (c) 2019 Elias Häußler <elias@haeussler.dev>. All rights reserved.
*/
const browser = require('browser-sync');
const {PATHS, SERVER} = require('../config');
module.exports = cb => {
browser.init({
server: PATHS.dist,
port: SERVER.port
});
if ("function" === typeof cb) cb();
};
/*
* Copyright (c) 2019 Elias Häußler <elias@haeussler.dev>. All rights reserved.
*/
const {series, watch} = require('gulp');
const {PATHS} = require('../config');
module.exports = cb => {
watch(PATHS.data, series('copy', 'reload'));
watch(PATHS.javascript.src, series('js:lint', 'js', 'reload'));
watch(PATHS.sass.src, series('sass:lint', 'sass', 'reload'));
watch(PATHS.pages, series('pages', 'reload'));
if ("function" === typeof cb) cb();
};
/*
* Copyright (c) 2018 Elias Häußler <mail@elias-haeussler.de> (www.elias-haeussler.de).
* Copyright (c) 2019 Elias Häußler <elias@haeussler.dev>. All rights reserved.
*/
// jshint node: true
'use strict';
const {parallel, series, task} = require('gulp');
const glob = require('glob-fs')();
const path = require('path');
const {PRODUCTION} = require('./gulp/config');
import plugins from 'gulp-load-plugins';
import yargs from 'yargs';
import gulp from 'gulp';
import browser from 'browser-sync';
import rimraf from 'rimraf';
import webpackStream from 'webpack-stream';
import webpack from 'webpack';
import named from 'vinyl-named';
// Register tasks from files
glob.readdirSync('./gulp/tasks/*.js').forEach(file => {
let name = path.basename(file, '.js').split(/(?=[A-Z])/).map(s => s.toLowerCase()).join(':');
task(name, require('./' + file));
});
// ==================== CONSTANTS ====================
// Define Server Settings
const SERVER = {
port: "8000"
};
// Define Paths
const PATHS = {
dist: "dist",
data: "src/data/**/*",
images: [
"src/assets/img/**/*"
],
javascript: {
all: "src/assets/js/**/*.js",
entry: "src/assets/js/main.js",
},
sass: {
all: "src/assets/scss/**/*.scss",
entry: "src/assets/scss/main.scss",
modules: [
"node_modules/reset-css",
"node_modules/spinkit/scss"
]
},
pages: "src/**/*.html"
};
// Browser compatibility
const COMPATIBILITY = [
"last 2 versions",
"ie >= 8",
"ios >= 7"
];
// Check for --production flag
const PRODUCTION = !!(yargs.argv.production);
// ==================== TASKS ====================
// Load all Gulp plugins
const $ = plugins();
/**
* SCSS Lint task
* @returns {*}
*/
let sass_lint = () =>
{
return gulp.src(PATHS.sass.all)
.pipe($.sassLint())
.pipe($.sassLint.format())
.pipe($.sassLint.failOnError());
};
/**
* Compile Sass
* @returns {*}
*/
let sass = () =>
{
return gulp.src(PATHS.sass.entry)
.pipe(named())
.pipe($.sourcemaps.init())
.pipe($.sass({
includePaths: [PATHS.sass.entry].concat(PATHS.sass.modules)
}).on('error', $.sass.logError))
.pipe($.autoprefixer({ browsers: COMPATIBILITY }))
.pipe($.if(PRODUCTION, $.cleanCss({
compatibility: 'ie8',
level: {
1: {
specialComments: 1
}
}
})))
.pipe($.if(!PRODUCTION, $.sourcemaps.write()))
.pipe(gulp.dest(PATHS.dist + '/assets/css'));
};
/**
* JavaScript Lint task
* @returns {*}
*/
let js_lint = () =>
{
return gulp.src(PATHS.javascript.all)
.pipe($.jshint())
.pipe($.jshint.reporter('default'));
};
/**
* Concatenate and minify JavaScript
* @returns {*}
*/
let javascript = () =>
{
return gulp.src(PATHS.javascript.entry)
.pipe(named())
.pipe($.sourcemaps.init())
.pipe(webpackStream(require('./webpack.config'), webpack))
.pipe($.if(!PRODUCTION, $.sourcemaps.write()))
.pipe(gulp.dest(PATHS.dist + '/assets/js'));
};
/**
* Copy pages to dist folder
* @returns {*}
*/
let pages = () =>
{
return gulp.src(PATHS.pages)
.pipe(gulp.dest(PATHS.dist));
};
/**
* Copy assets to dist folder
* @returns {*}
*/
let copy = () =>
{
return gulp.src(PATHS.data, { nodir: true })
.pipe(gulp.dest(PATHS.dist + '/data'));
};
/**
* Clean the dist folder
* @param done
*/
let clean = done =>
{
rimraf(PATHS.dist, done);
};
/**
* Start a Server with BrowserSync
* @param done
*/
let server = done =>
{
browser.init({
server: PATHS.dist, port: SERVER.port
});
done();
};
/**
* Reload Browser
* @param done
*/
let reload = done =>
{
browser.reload();
done();
};
/**
* Watch Files for Changes
*/
let watch = () =>
{
gulp.watch(PATHS.data, gulp.series(copy, reload));
gulp.watch(PATHS.javascript.all, gulp.series(js_lint, javascript, reload));
gulp.watch(PATHS.sass.all, gulp.series(sass_lint, sass, reload));
gulp.watch(PATHS.pages, gulp.series(pages, reload));
};
/**
* Build the dist folder
*/
let build = gulp.series(clean, gulp.parallel(copy, sass_lint, sass, js_lint, javascript, pages));
// Default Task
gulp.task('default', !PRODUCTION ? gulp.series(build, server, watch) : gulp.series(build));
// Server Task
gulp.task('serve', server);
// Watch Task
gulp.task('watch', watch);
// Register global tasks
task('build', series('clean', 'download', parallel('copy', 'sass', 'js', 'pages')));
task('lint', series('sass:lint', 'js:lint'));
task('serve', series('server', 'watch'));
task('default', series(...(PRODUCTION ? ['build'] : ['lint', 'build', 'serve'])));
{
"name": "student-growth",
"version": "2.1.0",
"name": "@eliashaeussler/student-growth",
"description": "Visualization about student growth in Germany",
"main": "gulpfile.js",
"scripts": {
"build": "./node_modules/.bin/gulp --production && ./node_modules/.bin/gulp serve",
"build": "NODE_ENV=prod ./node_modules/.bin/gulp",
"dev": "./node_modules/.bin/gulp",
"serve": "./node_modules/.bin/gulp serve",
"sources": "./node_modules/.bin/gulp --production",
"start": "./node_modules/.bin/gulp",
"test": "./node_modules/.bin/gulp --production",
"watch": "./node_modules/.bin/gulp serve && ./node_modules/.bin/gulp watch"
"test": "./node_modules/.bin/gulp lint"
},
"repository": "https://github.com/eliashaeussler/student-growth",
"author": "Elias Häußler <mail@elias-haeussler.de>",
"author": "Elias Häußler <elias@haeussler.dev>",
"license": "MIT",
"dependencies": {
"d3-array": "^1.2.1",
......@@ -30,31 +26,27 @@
"spinkit": "^1.2.5"
},
"devDependencies": {
"babel-core": "^6.26.3",
"babel-loader": "^7.1.4",
"babel-preset-env": "^1.7.0",
"@babel/core": "^7.4.4",
"@babel/preset-env": "^7.4.4",
"@babel/register": "^7.4.4",
"babel-loader": "^8.0.5",
"browser-sync": "^2.24.4",
"glob-fs": "^0.1.7",
"google-fonts-webpack-plugin": "^0.4.4",
"gulp": "gulpjs/gulp#4.0",
"gulp": "^4.0.1",
"gulp-autoprefixer": "^5.0.0",
"gulp-clean-css": "^3.9.4",
"gulp-if": "^2.0.2",
"gulp-jshint": "^2.1.0",
"gulp-load-plugins": "^1.5.0",
"gulp-sass": "^4.0.1",
"gulp-sass-lint": "^1.4.0",
"gulp-sourcemaps": "^2.6.4",
"js-yaml": "^3.13.1",
"jshint": "^2.9.5",
"rimraf": "^2.6.2",
"uglifyjs-webpack-plugin": "^1.2.7",
"vinyl-named": "^1.1.0",
"webpack": "^3.0",
"webpack-stream": "^4.0.3",
"yargs": "^11.0.0"
},
"babel": {
"presets": [
"env"
]
"webpack-stream": "^4.0.3"
}
}
/*
* Copyright (c) 2018 Elias Häußler <mail@elias-haeussler.de> (www.elias-haeussler.de).
* Copyright (c) 2019 Elias Häußler <elias@haeussler.dev>. All rights reserved.
*/
const GoogleFontsPlugin = require('google-fonts-webpack-plugin');
......@@ -10,11 +10,13 @@ module.exports = {
rules: [
{
test: /\.js$/,
use: [
{
loader: 'babel-loader'
exclude: /node_modules/,
use: {
loader: 'babel-loader',
options: {
presets: ['@babel/preset-env']
}
]
}
}
]
},
......
This source diff could not be displayed because it is too large. You can view the blob instead.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment