IPost/server.js

import "newrelic"

import http from "http";
import express,{Router} from "express";
import useragent from "express-useragent";
import fileUpload from "express-fileupload";
import * as bodyParser from "body-parser";
import cookieParser from "cookie-parser";
import * as mysql from "mysql";
import * as ws from "ws";
import getIP from "./extra_modules/getip.js";
import {unsign} from "./extra_modules/unsign.js";
import { readFileSync, appendFile } from "fs";
import { format } from "util";
import { setup as optionssetup } from "./routes/api/options.js";
import { setup as allsetup } from "./routes/api/all.js";
import { setup as settingshandlersetup } from "./routes/api/settingshandler.js";
import { setup as postsetup } from "./routes/api/post.js";
import { setup as dmsPersonalMessagessetup } from "./routes/api/dms/PersonalMessages.js";
import { setup as dmspostsetup } from "./routes/api/dms/post.js";
import { setup as fileiconsetup } from "./routes/api/getFileIcon.js";
import { setup as searchsetup } from "./routes/api/search.js";
import { setup as getpostssetup } from "./routes/api/getPosts.js";
import { setup as userroutessetup } from "./routes/api/userRoutes.js";
import { setup as servefilessetup} from "./routes/serve_static_files.js"
import { setup as userfilessetup} from "./routes/userfiles.js"
import { setup as userauthsetup} from "./routes/user_auth.js"

import { ensureExists } from "./extra_modules/ensureExists.js"

import * as compress from "compression"
const compression = compress.default

import { fileURLToPath } from 'url'
import { dirname } from 'path'
const __filename = fileURLToPath(import.meta.url)
const __dirname = dirname(__filename)

const config = JSON.parse(readFileSync("server_config.json"));
const time = Date.now();
const original_log = console.log;
/**
 * custom logging function
 * @param  {number}     level               importance level if information
 * @param  {any}        info                information to format + log
 * @return {undefined}                      returns nothing
 */
function log_info(level, ...info) {
    let text = info;
    if (text == undefined || text.length == 0) {
        text = level;
        level = 5;
    }
    if (config["logs"] && config["logs"]["level"] && config["logs"]["level"] >= level) {
        let tolog = `[INFO] [${Date.now()}] : ${format(text)} \n`;
        original_log(tolog); //still has some nicer colors
        ensureExists(__dirname + '/logs/', function (err) {
            if (err) {
                process.stderr.write(tolog); //just write it to stderr
            }
            else {
                appendFile(__dirname + "/logs/" + time, tolog, function (err) {
                    if (err) {
                        process.stderr.write(err);
                    }
                });
            }
        });
    }
}
console.log = log_info;

const WebSocket = ws.WebSocketServer;

const router = Router();
const app = express();
const DID_I_FINALLY_ADD_HTTPS = true;
const con = mysql.createPool({
    connectionLimit: config.mysql.connections,
    host: config.mysql.host,
    user: config.mysql.user,
    password: readFileSync(config.mysql.password_file).toString()
});
const cookiesecret = readFileSync("cookiesecret.txt").toString();
/**
 * quick function to convert data to base64
 * @param  {any} data               data to encode in base64
 * @return {string}      base64 encoded data
 */
function b64(data) {
    let buff = Buffer.from(data);
    return buff.toString('base64');
}
/**
 * custom, bad random number generator
 * @param       {number} seed  seed for the number generator, defaults to current timestamp
 * @constructor
 */
function RNG(seed) {
    if (!seed)
        seed = Date.now();
    this.seed = seed;
    this.random = function (min, max) {
        if (!min)
            min = 0;
        if (!max) {
            max = min;
            min = 0;
        }
        this.seed += Math.log(Math.abs(Math.sin(this.seed)) * 100);
        return Math.abs(Math.sin(this.seed)) * max + min;
    };
    this.rand = function (min, max) {
        return Math.floor(this.random(min, max));
    };
}
/**
 * waits x ms
 * @param  {number} ms               amount of ms to sleep for
 * @return {promise}    promise that gets resolved after x ms
 */
function sleep(ms) {
    return new Promise(resolve => setTimeout(resolve, ms));
}
const rand = new RNG();
const genstring_characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
const genstring_charactersLength = genstring_characters.length;
/**
 * generates a semi-random string
 * @param  {number} length               length of string to generate
 * @return {string}        semi-random string generated
 */
function genstring(length) {
    let result = "";
    for (let i = 0; i < length; i++) {
        result += genstring_characters.charAt(rand.rand(genstring_charactersLength));
    }
    return result;
}
/**
 * utility function to get a key by its value in an object
 * @param  {object}     object                  object to get key from
 * @param  {any}        value                   value to get key from
 * @return {any}                                key to the given value inside the object
 */
function getKeyByValue(object, value) {
    return Object.keys(object).find(key => object[key] === value);
}
var API_CALLS = {};
var API_CALLS_ACCOUNT = {};
var USER_CALLS = {};
var SESSIONS = {};
var REVERSE_SESSIONS = {};
var INDIVIDUAL_CALLS = {};
/**
 * clears current api call list (per IP)
 * @return {undefined} returns nothing
 */
function clear_api_calls() {
    API_CALLS = {};
}
/**
 * clears current api account call list (per account)
 * @return {undefined} returns nothing
 */
function clear_account_api_calls() {
    API_CALLS_ACCOUNT = {};
}
/**
 * clears current user file call list (per IP)
 * @return {undefined} returns nothing
 */
function clear_user_calls() {
    USER_CALLS = {};
}
setInterval(clear_api_calls, config.rate_limits.api.reset_time);
setInterval(clear_account_api_calls, config.rate_limits.api.reset_time);
setInterval(clear_user_calls, config.rate_limits.user.reset_time);
function increaseIndividualCall(url, req) {
    let conf = config["rate_limits"]["individual"][url];
    if (!conf) {
        //if(!url.startsWith("/avatars/")) //ignore avatars /* DEBUG: inidividual ratelimiters */
            //console.log(5, "url not in individual ratelimiter", url); /* DEBUG: inidividual ratelimiters */
        return true;
    }
    if (!conf["enabled"])
        return true;
    let ip = getIP(req);
    if (INDIVIDUAL_CALLS[ip] == undefined)
        INDIVIDUAL_CALLS[ip] = {};
    if (INDIVIDUAL_CALLS[ip][url] == undefined)
        INDIVIDUAL_CALLS[ip][url] = 0;
    if (INDIVIDUAL_CALLS[ip][url] == 0) {
        setTimeout(function () {
            INDIVIDUAL_CALLS[ip][url] = 0;
        }, conf["reset_time"]);
    }
    INDIVIDUAL_CALLS[ip][url]++;
    if (INDIVIDUAL_CALLS[ip][url] >= conf["max"]) {
        console.log(3, "ratelimiting someone on", url, INDIVIDUAL_CALLS[ip][url], conf["max"],ip);
        return false;
    }
    return true;
}
function increaseAccountAPICall(req, res) {
    let cookie = req.cookies.AUTH_COOKIE;
    if (!cookie) {
        return true;
    }
    let unsigned = unsign(cookie, req, res);
    if (!unsigned) {
        return true; //if there's no account, why not just ignore it
    }
    unsigned = decodeURIComponent(unsigned);
    if (!unsigned)
        return false;
    let values = unsigned.split(" ");
    let username = values[0];
    if (API_CALLS_ACCOUNT[username] == undefined)
        API_CALLS_ACCOUNT[username] = 0;
    if (API_CALLS_ACCOUNT[username] >= config.rate_limits.api.max_per_account) {
        res.status(429);
        res.send("You are sending way too many api calls!");
        return false;
    }
    return true;
}
function increaseAPICall(req, res, next) {
    let ip = getIP(req);
    if (API_CALLS[ip] == undefined)
        API_CALLS[ip] = 0;
    if (API_CALLS[ip] >= config.rate_limits.api.max_without_session) {
        if (REVERSE_SESSIONS[ip] && req.cookies.session !== REVERSE_SESSIONS[ip]) { //expected a session, but didn't get one
            res.status(429);
            res.send("You are sending way too many api calls!");
            return;
        }
        if (!req.cookies.session) {
            let session;
            do {
                session = genstring(300);
            } while (SESSIONS[session] != undefined);
            SESSIONS[session] = ip;
            REVERSE_SESSIONS[ip] = session;
            setTimeout(function () {
                SESSIONS[session] = undefined;
                REVERSE_SESSIONS[ip] = undefined;
            }, 50000);
            res.cookie('session', session, { maxAge: 100000, httpOnly: true, secure: DID_I_FINALLY_ADD_HTTPS });
            console.log(3, "sending session to " + ip);
        }
    }
    if (API_CALLS[ip] >= config.rate_limits.api.max_with_session) {
        res.status(429);
        res.send("You are sending too many api calls!");
        console.log(3, "rate limiting " + ip);
        return false;
    }
    API_CALLS[ip]++;
    if (!increaseAccountAPICall(req, res))
        return false; //can't forget account-based ratelimits
    if (next)
        next();
    return true;
}
function increaseUSERCall(req, res, next) {
    let ip = getIP(req);
    if (USER_CALLS[ip] == undefined)
        USER_CALLS[ip] = 0;
    if (USER_CALLS[ip] >= config.rate_limits.user.max) {
        res.status(429);
        res.send("You are sending too many requests!");
        console.log(2, "rate limiting " + ip);
        return false;
    }
    USER_CALLS[ip]++;
    if (next)
        next();
    return true;
}
console.log(5, "loading routes");
app.use(useragent.express());
app.use(fileUpload({
    limits: {
        files: 5,
        fileSize: 1_000_000
    }
}));

app.use(bodyParser.default.json({ limit: "100mb" }));
app.use(bodyParser.default.urlencoded({ limit: "100mb", extended: true }));
app.use(cookieParser(cookiesecret));
app.use(compression())
var blocked_headers = [
    'HTTP_VIA',
    'HTTP_X_FORWARDED_FOR',
    'HTTP_FORWARDED_FOR',
    'HTTP_X_FORWARDED',
    'HTTP_FORWARDED',
    'HTTP_CLIENT_IP',
    'HTTP_FORWARDED_FOR_IP',
    'VIA',
    'X_FORWARDED_FOR',
    'FORWARDED_FOR',
    'X_FORWARDED',
    'FORWARDED',
    'CLIENT_IP',
    'FORWARDED_FOR_IP',
    'HTTP_PROXY_CONNECTION'
];
if (!config.disallow_proxies_by_headers) {
    blocked_headers = [];
}
app.use(function (_req, res, next) {
    res.set("X-XSS-Protection", "1; mode=block");
    next();
});
if (DID_I_FINALLY_ADD_HTTPS) {
    //auto redirect to https
    app.use((req, res, next) => {
        if (req.secure) {
            //already secure
            next();
        }
        else {
            //redirect to https
            res.redirect('https://' + req.headers.host + req.url);
        }
    });
}
app.use("/*", function (req, res, next) {
    res.set("x-powered-by", "ipost");
    for (let i = 0; i < blocked_headers.length; i++) {
        if (req.header(blocked_headers[i]) != undefined) {
            res.json({ "error": "we don't allow proxies on our website." });
            return;
        }
    }
    let fullurl = req.baseUrl + req.path;
    if (fullurl != "/") {
        fullurl = fullurl.substring(0, fullurl.length - 1);
    }
    if (!increaseIndividualCall(fullurl, req)) {
        res.status(429);
        res.json({ "error": "you are sending too many requests!" });
        return;
    }
    next();
});
console.log(5, "finished loading user routes, starting with api routes");

const setuproute = handler => handler(router,con,commonfunctions)

/*

START /API/*

*/
var wss;
var commonfunctions = {
    increaseAPICall,
    increaseUSERCall,
    increaseAccountAPICall,
    increaseIndividualCall,
    wss,
    genstring,
    ensureExists,
    "dirname": __dirname,
    config,
    DID_I_FINALLY_ADD_HTTPS
};

setuproute(optionssetup)
setuproute(allsetup)
setuproute(settingshandlersetup)
const get_pid = setuproute(postsetup);
setuproute(dmsPersonalMessagessetup)
const get_dmpid = setuproute(dmspostsetup);
setuproute(fileiconsetup)
setuproute(searchsetup)
setuproute(getpostssetup)
setuproute(userroutessetup)


router.get("/api/getChannels",  function (_req, res) {
    res.set("Access-Control-Allow-Origin", "*");
    let sql = `select post_receiver_name from ipost.posts where post_is_private = '0' group by post_receiver_name;`;
    con.query(sql, [], function (err, result) {
        if (err)
            throw err;
        res.json(result);
    });
});
/*

END /API/*

*/

setuproute(servefilessetup)

router.get("/logout",  function (_req, res) {
    res.cookie("AUTH_COOKIE", "", { maxAge: 0, httpOnly: true, secure: DID_I_FINALLY_ADD_HTTPS });
    res.redirect("/");
});

let global_page_variables = {
    getPID: get_pid,
    getDMPID: get_dmpid,
}
commonfunctions.global_page_variables = global_page_variables
setuproute(userfilessetup) //needs getPID and getDMPID

setuproute(userauthsetup) //login & register

console.log(5, "finished loading routes");
app.use(router);
const httpServer = http.createServer(app);
httpServer.listen(config["ports"]["http"], function () {
    console.log(5, "HTTP Server is listening");
});
const privateKey = readFileSync(config["ssl"]["privateKey"]).toString();
const certificate = readFileSync(config["ssl"]["certificate"]).toString();
const credentials = { key: privateKey, cert: certificate };
var httpsServer;

import spdy from "spdy"

if (DID_I_FINALLY_ADD_HTTPS) {
    httpsServer = spdy.createServer(credentials,app)
    //httpsServer = https.createServer(credentials, app);
    httpsServer.listen(config["ports"]["https"], function () {
        console.log(5, "HTTPS Server is listening");
    });
}
else {
    httpsServer = httpServer;
}
wss = new WebSocket({
    server: httpsServer,
    perMessageDeflate: {
        zlibDeflateOptions: {
            chunkSize: 1024,
            memLevel: 7,
            level: 3
        },
        zlibInflateOptions: {
            chunkSize: 10 * 1024
        },
        clientNoContextTakeover: true,
        serverNoContextTakeover: true,
        serverMaxWindowBits: 10,
        concurrencyLimit: 10,
        threshold: 1024 * 16
    }
});
wss.on("connection", function connection(ws) {
    ws.channel = "everyone";
    console.log(5,"new connection");
    ws.on("message", function incoming(message) {
        message = JSON.parse(message);
        if (message.id == "switchChannel") {
            ws.channel = decodeURIComponent(message.data);
        }
    });
});
commonfunctions.wss = wss;
console.log(5, "starting up all services");