Nicolas/Livebeat
1
0
Fork 0
Code Issues 5 Pull Requests 1 Packages Releases Wiki Activity

Switch from RabbitMQ to server-sent events

Browse Source 
(not fully working yet)
This commit is contained in:
Nicolas Klier
2021-05-03 20:58:40 +02:00
parent 533749c7c8
commit daa7209742
38 changed files with 22158 additions and 672 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
backend/.idea/.gitignore generated vendored Normal file
View File

@@ -0,0 +1,8 @@
# Default ignored files
/shelf/
/workspace.xml
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml
# Editor-based HTTP Client requests
/httpRequests/

8
backend/.idea/backend.iml generated Normal file
View File

@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

6
backend/.idea/inspectionProfiles/profiles_settings.xml generated Normal file
View File

@@ -0,0 +1,6 @@
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>

4
backend/.idea/misc.xml generated Normal file
View File

@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.8" project-jdk-type="Python SDK" />
</project>

8
backend/.idea/modules.xml generated Normal file
View File

@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/backend.iml" filepath="$PROJECT_DIR$/.idea/backend.iml" />
</modules>
</component>
</project>

6
backend/.idea/vcs.xml generated Normal file
View File

@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$/.." vcs="Git" />
</component>
</project>

18
backend/app.ts
View File

@@ -7,13 +7,13 @@ import * as figlet from 'figlet';
import * as mongoose from 'mongoose';
import { exit } from 'process';
import * as winston from 'winston';
import { config } from './config';
import { GetBeat, GetBeatStats } from './endpoints/beat';
import { getNotification } from './endpoints/notification';
import { GetPhone, PostPhone } from './endpoints/phone';
import { DeleteUser, GetUser, LoginRabbitUser, LoginUser, MW_User, PatchUser, PostUser, Resource, Topic, VHost } from './endpoints/user';
import { DeleteUser, GetUser, LoginUser, MW_User, PatchUser, PostUser, UserEvents } from './endpoints/user';
import { hashPassword, randomPepper, randomString } from './lib/crypto';
import { EventManager } from './lib/eventManager';
import { RabbitMQ } from './lib/rabbit';
import { UserType } from './models/user/user.interface';
import { User } from './models/user/user.model';
@@ -28,6 +28,7 @@ export const IS_DEBUG = process.env.DEBUG == 'true';
export let logger: winston.Logger;
export let rabbitmq: RabbitMQ;
export let eventManager: EventManager = new EventManager();
async function run() {
const { combine, timestamp, label, printf, prettyPrint } = winston.format;
@@ -108,7 +109,7 @@ async function run() {
await User.create({
name: 'admin',
password: await hashPassword(randomPassword + salt + randomPepper()),
brokerToken: randomString(16),
eventToken: randomString(16),
salt,
createdAt: Date.now(),
lastLogin: 0,
@@ -139,15 +140,12 @@ async function run() {
app.get('/', (req, res) => res.status(200).send('OK'));
// User authentication
// User authentication & actions
app.post('/user/login', (req, res) => LoginUser(req, res));
app.get('/user/rabbitlogin', (req, res) => LoginRabbitUser(req, res));
app.get('/user/vhost', (req, res) => VHost(req, res));
app.get('/user/resource', (req, res) => Resource(req, res));
app.get('/user/topic', (req, res) => Topic(req, res));
// CRUD user
app.get('/user/notification', MW_User, (req, res) => getNotification(req, res)); // Notifications
app.get('/user/events', (req, res) => UserEvents(req, res));
app.get('/user/', MW_User, (req, res) => GetUser(req, res));
app.post('/user/', MW_User, (req, res) => PostUser(req, res));
app.get('/user/:id', MW_User, (req, res) => GetUser(req, res));
@@ -171,8 +169,8 @@ async function run() {
* Message broker
*/
rabbitmq = new RabbitMQ();
await rabbitmq.init();
logger.info("Connected with message broker.");
//await rabbitmq.init();
//logger.info("Connected with message broker.");
}
run();

4
backend/config.ts
View File

@@ -16,6 +16,10 @@ export const config: IConfig = {
host: "0.0.0.0"
}
}
/**
* END OF CONFIG
* ====================
*/
export interface IConfig {
authentification: {

75
backend/endpoints/beat.ts
View File

@@ -1,9 +1,13 @@
import { Response } from "express";
import { eventManager, logger } from "../app";
import { LivebeatRequest } from "../lib/request";
import { IBeat } from "../models/beat/beat.interface";
import { Beat } from "../models/beat/beat.model.";
import { ISeverity } from "../models/notifications/notification.interface";
import { Phone } from "../models/phone/phone.model";
const timeouts: Map<string, NodeJS.Timeout> = new Map<string, NodeJS.Timeout>();
export async function GetBeatStats(req: LivebeatRequest, res: Response) {
const phones = await Phone.find({ user: req.user?._id });
const perPhone: any = {};
@@ -20,27 +24,88 @@ export async function GetBeatStats(req: LivebeatRequest, res: Response) {
}
export async function GetBeat(req: LivebeatRequest, res: Response) {
const from: number = Number(req.query.from);
const to: number = Number(req.query.to);
const from: number = Number(req.query.from || 0);
const to: number = Number(req.query.to || Date.now() / 1000);
const limit: number = Number(req.query.limit || 10000);
const sort: number = Number(req.query.sort || 1); // Either -1 or 1
const phoneId = req.query.phoneId;
// Grab default phone if non was provided.
const phone = req.query.phone === undefined ? await Phone.findOne({ user: req.user?._id }) : await Phone.findOne({ _id: phoneId, user: req.user?._id });
let beats: IBeat[] = []
let beats: IBeat[] = [];
//console.log(from, to);
//console.log(`Search from ${new Date(from).toString()} to ${new Date(to * 1000).toString()}`);
if (phone !== null) {
beats = await Beat.find(
{
phone: phone._id,
createdAt: {
$gte: new Date((from | 0) * 1000),
$lte: new Date((to | Date.now() /1000) * 1000)
$gte: new Date((from)),
$lte: new Date(to * 1000)
}
}).sort({ _id: sort }).limit(limit);
res.status(200).send(beats);
} else {
res.status(404).send({ message: 'Phone not found' });
}
}
export async function AddBeat(req: LivebeatRequest, res: Response) {
const beat = req.body as IBeat;
const androidId = req.headers.deviceId as string;
if (androidId === undefined) {
res.status(401).send({ message: 'Device id is missing' });
}
// Get phone
const phone = await Phone.findOne({ androidId });
if (phone == undefined) {
logger.warning(`Received beat from unknown device with id ${androidId}`);
return;
}
let newBeat;
if (beat.coordinate !== undefined && beat.accuracy !== undefined) {
logger.info(`New beat from ${phone.displayName} => ${beat.coordinate[0]}, ${beat.coordinate[1]} | Height: ${beat.coordinate[3]}m | Speed: ${beat.coordinate[4]} | Accuracy: ${beat.accuracy}% | Battery: ${beat.battery}%`);
newBeat = await Beat.create({
phone: phone._id,
// [latitude, longitude, altitude]
coordinate: [beat.coordinate[0], beat.coordinate[1], beat.coordinate[2]],
accuracy: beat.coordinate[3],
speed: beat.coordinate[4],
battery: beat.battery,
createdAt: Date.now()
});
}
newBeat = await Beat.create({
phone: phone._id,
battery: beat.battery,
createdAt: Date.now()
});
// Broadcast if device became active
if (timeouts.has(phone.id)) {
clearTimeout(timeouts.get(phone.id)!!);
} else {
phone.active = true;
await phone.save();
eventManager.push('phone_alive', phone.toJSON(), phone.user);
}
const timeoutTimer = setTimeout(async () => {
eventManager.push('phone_dead', phone.toJSON(), phone.user, ISeverity.WARN);
timeouts.delete(phone.id);
phone.active = false;
await phone.save();
}, 60_000);
timeouts.set(phone.id, timeoutTimer);
eventManager.push('beat', newBeat.toJSON(), phone.user);
}

1
backend/endpoints/phone.ts
View File

@@ -5,7 +5,6 @@ import { Beat } from "../models/beat/beat.model.";
import { Phone } from "../models/phone/phone.model";
export async function GetPhone(req: LivebeatRequest, res: Response) {
const phoneId: String = req.params['id'];

180
backend/endpoints/user.ts
View File

@@ -1,7 +1,7 @@
import { Request, Response } from 'express';
import { decode, sign, verify } from 'jsonwebtoken';
import { JWT_SECRET, logger, RABBITMQ_URI } from '../app';
import { eventManager, JWT_SECRET, logger, RABBITMQ_URI } from '../app';
import * as jwt from 'jsonwebtoken';
import { config } from '../config';
import { hashPassword, randomPepper, randomString, verifyPassword } from '../lib/crypto';
@@ -51,7 +51,7 @@ export async function PostUser(req: LivebeatRequest, res: Response) {
}
const salt = randomString(config.authentification.salt_length);
const brokerToken = randomString(16);
const eventToken = randomString(16);
const hashedPassword = await hashPassword(password + salt + randomPepper()).catch(error => {
res.status(400).send({ message: 'Provided password is too weak and cannot be used.' });
return;
@@ -61,7 +61,7 @@ export async function PostUser(req: LivebeatRequest, res: Response) {
name,
password: hashedPassword,
salt,
brokerToken,
eventToken,
type,
lastLogin: new Date(0)
});
@@ -72,6 +72,27 @@ export async function PostUser(req: LivebeatRequest, res: Response) {
res.status(200).send({ setupToken });
}
export async function UserEvents(req: LivebeatRequest, res: Response) {
if (req.query.token === undefined) {
res.status(401).send({ message: 'You need to define your event token.' });
return;
}
const eventToken = req.query.token as string;
const user = await User.findOne({ eventToken });
if (user === null) {
res.status(401).send({ message: 'This event token is not valid.' });
return;
}
eventManager.join(user.id, res);
}
export async function UserSubscribeEvent(req: Request, res: Response) {
}
export async function DeleteUser(req: Request, res: Response) {
}
@@ -120,159 +141,6 @@ export async function LoginUser(req: Request, res: Response) {
res.status(200).send({ token });
}
/**
* This function handles all logins to RabbitMQ since they need a differnt type of response
* then requests from frontends (web and phone).
*/
export async function LoginRabbitUser(req: Request, res: Response) {
const username = req.query.username;
const password = req.query.password;
res.status(200);
if (username === undefined || password === undefined) {
res.send('deny');
return;
}
// Check if request comes from backend. Basicly, we permitting ourself to connect with RabbitMQ.
if (username === "backend" && password === RABBITMQ_URI.split(':')[2].split('@')[0]) {
res.send('allow administrator');
return;
}
// Get user from database
const user = await User.findOne({ name: username.toString() });
// If we are here, it means we have a non-admin user.
if (user === null) {
res.send('deny');
return;
}
// Auth token for message broker is stored in plain text since it's randomly generated and only grants access to the broker.
if (user.brokerToken === password.toString()) {
if (user.type === UserType.ADMIN) {
res.send('allow administrator');
} else {
// Not an admin, grant user privilieges
res.send('allow user')
}
return;
}
res.send('deny');
}
/**
* This function basicly allows access to the root vhost if the user is known.
*/
export async function VHost(req: Request, res: Response) {
const vhost = req.query.vhost;
const username = req.query.username;
if (vhost === undefined || username === undefined) {
res.status(200).send('deny');
return;
}
if (vhost != '/') {
res.status(200).send('deny');
return;
}
// Check if user is us
if (username === 'backend') {
res.status(200).send('allow');
return;
}
const user = await User.findOne({ name: username.toString() });
if (user === null) {
// Deny if user doesn't exist.
res.status(200).send('deny');
} else {
res.status(200).send('allow');
}
}
export async function Resource(req: Request, res: Response) {
const username = req.query.username;
const vhost = req.query.vhost;
const resource = req.query.resource;
const name = req.query.name;
const permission = req.query.permission;
const tags = req.query.tags;
if (username === undefined || vhost === undefined || resource === undefined || name === undefined || permission === undefined || tags === undefined) {
res.status(200).send('deny');
return;
}
// Check if it's us
if (username.toString() == 'backend') {
res.status(200).send('allow');
return;
}
// Deny if not root vhost
if (vhost.toString() != '/') {
res.status(200).send('deny');
return;
}
// Check if user exists
const user = await User.findOne({ name: username.toString() });
if (user == null) {
res.status(200).send('deny');
return;
}
if (tags.toString() == "administrator" && user.type != UserType.ADMIN) {
res.status(200).send('deny');
return;
}
// TODO: This has to change if we want to allow users to see the realtime movement of others.
if (resource.toString().startsWith('tracker-') && resource != 'tracker-' + username) {
res.status(200).send('deny');
return;
}
res.status(200).send('allow');
}
export async function Topic(req: Request, res: Response) {
res.status(200);
const username = req.query.username;
const routingKey = req.query.routing_key;
if (routingKey === undefined || username === undefined) {
res.send('deny');
return;
}
// Check if it's us
if (username.toString() == 'backend') {
res.status(200).send('allow');
return;
}
// Check if user exists
const user = await User.findOne({ name: username.toString() });
if (user === null) {
res.send('deny');
return;
}
if (routingKey !== user.id) {
res.send('deny');
return;
}
res.status(200).send('allow');
}
/**
* This middleware validates any tokens that are required to access most of the endpoints.
* Note: This validation doesn't contain any permission checking.

170
backend/lib/eventManager.ts Normal file
View File

@@ -0,0 +1,170 @@
import { Response } from "express";
import { logger } from "../app";
import { ISeverity, NotificationType, PublicNotificationType } from "../models/notifications/notification.interface";
import { addNotification } from "../models/notifications/notification.model";
import { IPhone } from "../models/phone/phone.interface";
import { IUser } from "../models/user/user.interface";
import { User } from "../models/user/user.model";
import { randomString } from "./crypto";
/**
* This class stores one specific client.
*/
export class Client {
id: string;
userId: string;
stream: Response;
constructor(stream: Response, userId: string) {
this.id = randomString(16);
this.userId = userId;
this.stream = stream;
}
send(type: NotificationType, data: any) {
this.stream.write(`event: ${type}\ndata: ${JSON.stringify(data)}\n\n`);
}
async getUser() {
return await User.findById(this.userId);
}
}
export class Clients {
private clients: Client[];
constructor(clients: Client[]) {
this.clients = clients;
}
getClientsByUser(userId: string) {
const userClients = [];
for (let i = 0; i < this.clients.length; i++) {
if (this.clients[i].userId === userId) {
userClients.push(this.clients[i]);
}
}
return userClients;
}
closeAllClientsByUser(userId: string) {
this.getClientsByUser(userId).forEach(client => {
client.stream.end();
});
}
addClient(client: Client) {
this.clients.push(client);
return client;
}
getClients() {
return this.clients;
}
}
export class EventManager {
constructor() {
setInterval(() => {
this.broadcast('info', { message: "Test" });
}, 2000);
}
// This map stores a open data stream and it's associated room.
private clients: Clients = new Clients([]);
private addClient(stream: Response, userId: string) {
this.clients.addClient(new Client(stream, userId));
}
/**
* Add a client to a specific room
* @param room Used as an id for the specific room
* @param stream A open connection to the user
*/
async join(userId: string, stream: Response) {
if (stream.req == undefined) {
stream.send(500);
return;
}
// Check user
const user = await User.findById(userId);
if (user === null) {
stream.send(401);
return;
}
// Make sure to keep the connection open
stream.writeHead(200, {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive'
});
this.addClient(stream, userId);
logger.debug(`Client ${stream.req.hostname} of user ${user.name} joined.`);
}
/**
* Push a new event into a specific room
* @param event Type of the event
* @param data Content of the event
* @param selector Room to push in. If empty then it will be a public broadcast to anyone.
*/
push(type: NotificationType, data: any, user: IUser, severity = ISeverity.INFO) {
let clients = this.clients.getClientsByUser(user.id);
if (clients === undefined) return;
/* Manage notifications */
if (type != 'beat' && user !== undefined) {
if (type == 'phone_alive' || type == 'phone_dead') {
addNotification(type, severity, ((data as IPhone)._id), user);
}
}
data = { type, severity, ...data };
clients.forEach((client) => {
client.stream.write(`event: ${type}\ndata: ${JSON.stringify(data)}\n\n`);
});
if (user === undefined) {
logger.debug(`Broadcasted event ${type} to all users (${clients.length} clients affected)`);
} else {
logger.debug(`Broadcasted event ${type} to user ${user.id} (${clients.length} clients affected)`);
}
}
/**
* Very much like push() but it will send this message to **every connected client!**
*/
broadcast(type: PublicNotificationType, data: any) {
this.clients.getClients().forEach(async client => {
console.log(`Send ${JSON.stringify(data)} (of type ${type}) to a client of user ${(await client.getUser())?.name}`);
client.stream.write(`event: message\ndata: ${JSON.stringify(data)}\n\n`);
});
logger.debug(`Broadcasted event ${type} to all users (${this.clients.getClients().length} clients affected)`);
}
/**
* End the communication with a specific client.
*/
end(stream: Response, userId: string) {
stream.end();
logger.debug(`End connection with ${stream.req?.hostname} (user: ${userId})`);
}
static buildEventTypeName(type: EventType, user: IUser) {
return `${type}-${user}`;
}
}
export type EventType =
| 'tracker' // Receive just the gps location of a specific user.
| 'user' // Receive user updates.
| 'all'; // Receive all above events.

2
backend/lib/rabbit.ts
View File

@@ -63,7 +63,7 @@ export class RabbitMQ {
this.timeouts.delete(phone.id);
phone.active = false;
await phone.save();
}, 30_000);
}, 60_000);
this.timeouts.set(phone.id, timeoutTimer);
this.publish(phone.user.toString(), newBeat.toJSON(), 'beat');

4
backend/models/beat/beat.interface.ts
View File

@@ -4,8 +4,8 @@ import { IPhone } from '../phone/phone.interface';
export interface IBeat extends Document {
// [latitude, longitude, altitude, accuracy, speed]
coordinate?: number[],
accuracy: number,
speed: number,
accuracy?: number,
speed?: number,
battery?: number,
phone: IPhone,
createdAt?: Date

3
backend/models/notifications/notification.interface.ts
View File

@@ -8,7 +8,8 @@ export enum ISeverity {
ERROR = 3
}
export type NotificationType = 'beat' | 'phone_alive' | 'phone_dead' | 'phone_register' | 'panic';
export type NotificationType = 'beat' | 'phone_alive' | 'phone_dead' | 'phone_register' | 'panic' | 'test';
export type PublicNotificationType = 'shutdown' | 'restart' | 'warning' | 'error' | 'info';
export interface INotification extends Document {
type: NotificationType;

2
backend/models/user/user.interface.ts
View File

@@ -13,6 +13,6 @@ export interface IUser extends Document {
type: UserType,
lastLogin: Date,
twoFASecret?: string,
brokerToken: string,
eventToken: string,
createdAt?: Date
}

2
backend/models/user/user.schema.ts
View File

@@ -6,7 +6,7 @@ const schemaUser = new Schema({
salt: { type: String, required: true },
type: { type: String, required: true, default: 'user' }, // This could be user, admin, guest
twoFASecret: { type: String, required: false },
brokerToken: { type: String, required: true },
eventToken: { type: String, required: true },
lastLogin: { type: Date, required: true, default: Date.now },
}, {
timestamps: {

3774
backend/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

11
backend/package.json
View File

@@ -20,6 +20,7 @@
"@types/node": "^14.14.9",
"amqplib": "^0.6.0",
"argon2": "^0.27.0",
"bi-directional-map": "^1.0.0",
"body-parser": "^1.19.0",
"chalk": "^4.1.0",
"cors": "^2.8.5",
@@ -34,20 +35,20 @@
"winston": "^3.3.3"
},
"devDependencies": {
"@types/amqplib": "0.5.14",
"@types/argon2": "0.15.0",
"@types/body-parser": "1.19.0",
"@types/chalk": "2.2.0",
"@types/cors": "2.8.8",
"@types/dotenv": "8.2.0",
"@types/express": "4.17.8",
"@types/figlet": "1.2.0",
"@types/jsonwebtoken": "8.5.0",
"@types/moment": "2.13.0",
"@types/mongoose": "5.7.36",
"@types/typescript": "2.0.0",
"@types/winston": "2.4.4",
"concurrently": "^5.3.0",
"nodemon": "^2.0.5",
"@types/jsonwebtoken": "8.5.0",
"@types/amqplib": "0.5.14",
"@types/cors": "2.8.8",
"@types/moment": "2.13.0"
"nodemon": "^2.0.5"
}
}