From ebe288a6321ea060d00a34aa574cd8bd16c8d629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=ADna=20Oltmanov=C3=A1?= Date: Mon, 15 May 2023 20:10:06 +0200 Subject: [PATCH 1/2] NERD updated to provide API v2 for Vue.js FE The committed code was part of the master's thesis that created modern web interface for NERD system. The changes are needed in order to provide new endpoints that FE can use. All previous functionality remains intact. --- NERDweb/api_v2.py | 1083 ++++++++++++++++++++++++++++++ NERDweb/api_v2_swag/login.yml | 29 + NERDweb/api_v2_swag/register.yml | 37 + NERDweb/api_v2_swag/search.yml | 161 +++++ NERDweb/auth.py | 51 ++ NERDweb/nerd_main.py | 252 +++++++ NERDweb/user_management.py | 127 ++++ NERDweb/userdb.py | 156 ++++- 8 files changed, 1892 insertions(+), 4 deletions(-) create mode 100644 NERDweb/api_v2.py create mode 100644 NERDweb/api_v2_swag/login.yml create mode 100644 NERDweb/api_v2_swag/register.yml create mode 100644 NERDweb/api_v2_swag/search.yml create mode 100644 NERDweb/auth.py create mode 100644 NERDweb/user_management.py diff --git a/NERDweb/api_v2.py b/NERDweb/api_v2.py new file mode 100644 index 00000000..380774f4 --- /dev/null +++ b/NERDweb/api_v2.py @@ -0,0 +1,1083 @@ +from flask import Blueprint, flash, redirect, session, g, make_response, \ + current_app, url_for, request, make_response, Response, jsonify, url_for +import requests +from flask_pymongo import pymongo, PyMongo +from datetime import datetime, timedelta, timezone +import jwt +from jwt import PyJWKClient +from jwt.exceptions import DecodeError +from werkzeug.exceptions import InternalServerError, Unauthorized +import os +import sys +import common.config +from auth import token_required +from common.utils import ipstr2int, int2ipstr, parse_rfc_time +import json +from bcrypt import hashpw, checkpw, gensalt +import subprocess +import re +from flasgger import swag_from +from flask_mail import Mail, Message +from requests_oauthlib import OAuth1Session, OAuth2Session + +from user_management import get_hashed_password, generate_token, verify_email_token, \ + generate_jwt_token, confirm_jwt_token +from nerd_main import get_ip_info, get_basic_info_dic, create_query_v2, \ + find_ip_data, get_basic_info_dic_v2, attach_whois_data, mailer, mongo, \ + get_ip_blacklists, ip_to_warden_data +from userdb import get_user_info, authenticate_with_token, generate_unique_token, \ + get_user_by_id, create_user, verify_user, set_last_login, get_user_name, \ + set_new_password, get_users_admin, set_verification_email_sent, set_new_roles, \ + delete_user, just_verify_user_by_id, set_api_v1_token, get_user_by_email + +DEFAULT_CONFIG_FILE = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "/etc/nerd/nerdweb.yml")) + +if len(sys.argv) >= 2: + cfg_file = sys.argv[1] +else: + cfg_file = DEFAULT_CONFIG_FILE +cfg_dir = os.path.dirname(os.path.abspath(cfg_file)) + +# Read web-specific config (nerdweb.cfg) +config = common.config.read_config(cfg_file) +# Read common config (nerd.cfg) and combine them together +common_cfg_file = os.path.join(cfg_dir, config.get('common_config')) +config.update(common.config.read_config(common_cfg_file)) + +config.testing = True + +# url for redirects +APP_BASE_URL = config.get('app_base_url') + +api_v2 = Blueprint("api_v2", __name__, static_folder="static", template_folder="templates") + +def check_email(email): + return re.fullmatch(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,7}\b', email) + +def send_password_reset_email(user_email): + if not config.get('login.request-email', None): + return dict(message='Error while sendig email.'), 402 + token = generate_jwt_token(user_email) + password_reset_url = url_for('api_v2.password_reset', token=token, _external=True) + msg = Message(subject="[NERD] Password reset request", + recipients=[user_email], + reply_to=current_app.config.get('MAIL_DEFAULT_SENDER'), + body=f"Dear user,\n\nyou can reset your password by clicking on this link:" + f"\n\n{password_reset_url}\n\nThank you,\nNERD administrator", + ) + mailer.send(msg) + +######################################################## +# NERD ENDPOINTS # +######################################################## + +# ***** NERD API BasicInfo ***** +@api_v2.route('/ip/', methods=['GET']) +def get_basic_info_v2(ipaddr=None): + """Basic info for a single IP address + --- + parameters: + - name: ipaddr + in: path + type: string + required: true + description: IP address in IPv4 format + responses: + 200: + description: An object containing basic info about IP + """ + ret, val = get_ip_info(ipaddr, False) + if not ret: + return val # val is an error Response + + binfo = get_basic_info_dic(val) + + return Response(json.dumps(binfo), 200, mimetype='application/json') + +# ***** NERD API IPSearch ***** +@api_v2.route('/search/ip', methods=['POST']) +@swag_from('./api_v2_swag/search.yml', validation=True) +def ip_search_v2(full = False): + err = {} + + # Get output format + output = request.json + query = create_query_v2(output) + + if "page" not in output: + output["page"] = 1 + + if "limit" not in output: + output["limit"] = 20 + + if "order" not in output: + output["order"] = "desc" + + if "sort" not in output or output["sort"] is None: + output["sort"] = "rep" + elif output["sort"] == "ip": + output["sort"] = "_id" + + try: + results = find_ip_data(query, (output["page"] - 1) * output["limit"], output["limit"]) # note: limit=0 means no limit + results.sort(output["sort"], 1 if output["order"] == "asc" else -1) + results = list(results) + except pymongo.errors.ServerSelectionTimeoutError: + log_err.log('503_db_error') + err['err_n'] = 503 + err['error'] = 'Database connection error' + resp = Response(json.dumps(err), 503, mimetype='application/json') + return resp + + # Return results + if output == "list": + resp = Response(''.join(int2ipstr(res['_id'])+'\n' for res in results), 200, mimetype='text/plain') + return resp + + # Convert _id from int to dotted-decimal string + for res in results: + res['_id'] = int2ipstr(res['_id']) + + lres = [] + if output == "short": + for res in results: + lres.append(get_basic_info_dic_short(res)) + else: + for res in results: + attach_whois_data(res, full) + lres.append(get_basic_info_dic_v2(res)) + + resp = Response(json.dumps(lres, default=str), 200, mimetype='application/json') + return resp + +# ***** NERD API Details ***** +@api_v2.route('/details/', methods=['GET']) +def get_full_info(ipaddr=None): + """Detailed info for a single IP address + --- + parameters: + - name: ipaddr + in: path + type: string + required: true + description: IP address in IPv4 format + responses: + 200: + description: An object containing detailed info about IP + """ + ret, val = get_ip_info(ipaddr, True) + if not ret: + return val # val is an error Response + + data = { + 'ip' : val['_id'], + 'rep' : val.get('rep', 0.0), + 'fmp' : val.get('fmp', {'general': 0.0}), + 'hostname' : (val.get('hostname', '') or '')[::-1], + 'ipblock' : val.get('ipblock', ''), + 'bgppref' : val.get('bgppref', ''), + 'asn' : val.get('asn',[]), + 'geo' : val.get('geo', None), + 'ts_added' : val['ts_added'].strftime("%Y-%m-%dT%H:%M:%S"), + 'ts_last_update' : val['ts_last_update'].strftime("%Y-%m-%dT%H:%M:%S"), + 'last_activity' : val['last_activity'].strftime("%Y-%m-%dT%H:%M:%S") if 'last_activity' in val else None, + 'bl' : [ { + 'name': bl['n'], + 'last_check': bl['t'].strftime("%Y-%m-%dT%H:%M:%S"), + 'last_result': True if bl['v'] else False, + 'history': [t.strftime("%Y-%m-%d") for t in bl['h']] + } for bl in val.get('bl', []) ], + 'events' : ip_to_warden_data(ipaddr), + 'misp_events' : val.get('misp_events', []), + 'events_meta' : { + 'total': val.get('events_meta', {}).get('total', 0.0), + 'total1': val.get('events_meta', {}).get('total1', 0.0), + 'total7': val.get('events_meta', {}).get('total7', 0.0), + 'total30': val.get('events_meta', {}).get('total30', 0.0), + }, + 'dshield' : val.get('dshield', []), + 'otx_pulses' : val.get('otx_pulses', []), + 'tags': val.get('tags', []), + + } + + return Response(json.dumps(data, default=str), 200, mimetype='application/json') + + + +######################################################## +# USER ENDPOINTS # +######################################################## +@api_v2.route('/login/devel') +def login_devel_v2(): + if not config.testing: + return flask.abort(404) + out = {} + out['user'] = { + 'login_type': 'devel', + 'id': 'devel_admin', + } + out['exp'] = datetime.utcnow()+timedelta(hours=4) + encoded_jwt = jwt.encode(out, config.get('secret_key'), algorithm="HS256") + return encoded_jwt + +# User login +@api_v2.route('/login', methods=['POST']) +@swag_from('./api_v2_swag/login.yml', validation=True) +def login_user_v2(): + try: + data = request.json + if not data: + return { + "message": "Please provide user details", + "data": None, + "error": "Bad request" + }, 400 + user = get_user_by_id("local:" + data["email"]) + + if user is None: + user = get_user_by_email(data["email"]) + if user is None: + return dict(message='User does not exist.'), 404 + return dict(message='Local user does not exist but provided email is associated with provider: ' + user['id'].split(":")[0]), 404 + + if not checkpw(data["password"].encode('utf-8'), user['password'].encode('utf-8')): + return dict(message='Bad password.'), 400 + try: + out = {} + out['user'] = { + 'login_type': 'local', + 'id': user['id'], + 'email': user['email'], + } + out['exp'] = datetime.utcnow()+timedelta(hours=4) + token = jwt.encode(out, config.get('secret_key'), algorithm="HS256") + out2 = {} + out2['user'] = { + 'login_type': 'local', + 'id': user['id'], + 'email': user['email'], + } + out2['exp'] = datetime.utcnow()+timedelta(hours=72) + refreshToken = jwt.encode(out2, config.get('secret_key'), algorithm="HS256") + set_last_login(datetime.utcnow(), user['id']) + return Response(json.dumps([token, refreshToken], default=str), 200, mimetype='application/json') + + except Exception as e: + return { + "error": "Something went wrong", + "message": str(e) + }, 500 + return { + "message": "Error fetching auth token!, invalid email or password", + "data": None, + "error": "Unauthorized" + }, 404 + except Exception as e: + return { + "message": "Something went wrong!", + "error": str(e), + "data": None + }, 500 + +@api_v2.route('/register', methods=['POST']) +@swag_from('./api_v2_swag/register.yml', validation=True) +def register_user_v2(): + data = request.json + if not check_email(data["email"]): + return dict(message='Wrong email format.'), 400 + if len(data["password"]) < 8: + return dict(message='Password not long enough.'), 400 + + hashed_password = get_hashed_password(data["password"]) + res = create_user(data["email"], hashed_password, "local", data["name"], data["organization"]) + if isinstance(res, Exception): + if res.args[0].startswith("duplicate key") and "Key (id)" in res.args[0]: #TODO? neexistuje lep�� zp�sob kontroly typu cyhby? Pr� na to speci�ln� typ Exc asi nen� + return dict(message=f"User with email address {data['email']} already exists! You can either log in or try to " + f"reset your password in log-in section."), 400 + else: + return dict(message=f"ERROR in register_user(): Something has failed during registration process: {res.args}"), 400 + else: + out = {} + out['user'] = { + 'email': data['email'], + } + out['exp'] = datetime.utcnow()+timedelta(hours=24) + encoded_jwt = jwt.encode(out, config.get('secret_key'), algorithm="HS256") + msg = Message(subject="[NERD] Account created", + recipients=[data["email"]], + reply_to=current_app.config.get('MAIL_DEFAULT_SENDER'), + ) + msg.body = f"NERD - Network Entity Reputation Database \nEmail address verification \n>Dear user {data['email']}, \nyou can activate your NERD account by clicking the link below (the link is valid for 24 hours): \n{config.get('email_web')}/verify?accessToken={encoded_jwt} \nNERD administrator" + msg.html = f"

NERD

Network Entity Reputation Database

Email address verification

Dear user {data['email']},

you can activate your NERD account by clicking this LINK (the link is valid for 24 hours).


NERD administrator

" + mailer.send(msg) + set_verification_email_sent(datetime.utcnow(), "local:" + data["email"]) + return dict(message=f"Account created for {data['email']}!"), 200 + +@api_v2.route('/verify', methods=['POST']) +def verify_email_address(): + """Email verification after registration + --- + consumes: + - application/json + parameters: + - in: body + name: body + description: Request body for endpoint + required: true + schema: + type: object + properties: + accessToken: + type: string + description: Email token provided after user registration + example: "ey...." + responses: + 200: + description: Successful email verification + 400: + description: accessToken missing + 500: + description: Internal server error + """ + data = request.json + if not data["accessToken"]: + return dict(message='No access token.'), 400 + try: + out = jwt.decode(data["accessToken"], config.get('secret_key'), algorithms=["HS256"]) + except Exception as e: + return { + "message": "Something went wrong", + "data": None, + "error": str(e) + }, 500 + try: + verify_user("local:" + out["user"]["email"]) + except Exception as e: + return { + "message": "Something went wrong", + "data": None, + "error": str(e) + }, 500 + return dict(message=f"Email verified for {out['user']['email']}!"), 200 + + +# gets user info extracted from JWT +@api_v2.route('/me', methods=['GET']) +@token_required +def me_info(current_user): + """User profile info + --- + security: + - OAuth2: [user] + responses: + 200: + description: User info fetched + """ + return jsonify({ "email": current_user["email"], "groups": current_user["groups"], "name": current_user["name"], "org": current_user["org"], "type": current_user["id"].split(":")[0], "api_v1_token": current_user["api_token"]}), 200 + + +@api_v2.route('/reset_password', methods=['POST']) +def password_reset_request(): + """Local user password reset request + --- + consumes: + - application/json + parameters: + - in: body + name: body + description: Request body for endpoint + required: true + schema: + type: object + properties: + email: + type: string + description: Local user email + example: "user@email.com" + responses: + 200: + description: Password reset email sent + """ + data = request.json + user = get_user_by_id("local:" + data["email"]) + if user is None: + return dict(message='User with this email does not exist.'), 401 + else: + send_password_reset_email(data["email"]) + return dict(message='Email with password reset link was sent!'), 200 + +@api_v2.route("/password_reset_request/", methods=['GET']) +def password_reset(token): + """Password reser redirect URI form email + --- + parameters: + - name: token + in: path + type: string + required: true + description: Token form email + responses: + 401: + description: User not found + """ + user = confirm_jwt_token(token) + if user is None or user['email'] is None: + return dict(message='User not found'), 401 + + return redirect(APP_BASE_URL + "/nerd2/password-reset?token=" + token) + +@api_v2.route("/password_reset_from_token", methods=['POST']) +def password_reset_token(): + """Local user password reset action + --- + consumes: + - application/json + parameters: + - in: body + name: body + description: Request body for endpoint + required: true + schema: + type: object + properties: + token: + type: string + description: Token from email + example: "ey..." + responses: + 200: + description: Password has been successfully changed + 400: + description: Problem when reseting password + 404: + description: User not found + """ + data = request.json + token = data["token"] + user = confirm_jwt_token(token) + if user is None or user['email'] is None: + return dict(message='User not found'), 404 + + if len(data["password"]) < 8: + return dict(message='Password not long enough'), 400 + + result = set_new_password(get_hashed_password(data["password"]), "local:" + user['email']) + if result is not None: + # exception occurred + return dict(message="Password reset failed. Please, try again and if the problem persists, contact NERD administrator."), 400 + else: + return dict(message="Password has been successfully changed!"), 200 + + +@api_v2.route("/change_password", methods=['POST']) +@token_required +def password_change(current_user): + """Local user password change + --- + security: + - OAuth2: [user] + consumes: + - application/json + parameters: + - in: body + name: body + description: Request body for endpoint + required: true + schema: + type: object + properties: + passOld: + type: string + description: old passowrd + example: "password123" + password: + type: string + description: new password + example: "password12" + responses: + 200: + description: Password has been successfully changed + 400: + description: Problem when reseting password + 404: + description: Wrong user or password + """ + data = request.json + + if len(data["password"]) < 8: + return dict(message='Password not long enough'), 404 + + user = get_user_by_id("local:" + current_user["email"]) + + if user is None: + return dict(message='User does not exist.'), 404 + + try: + if not checkpw(data["passOld"].encode('utf-8'), user["password"].encode('utf-8')): + return dict(message='Bad password.'), 404 + except Exception as e: + return { + "message": "Something went wrong!", + "error": str(e), + "data": None + }, 500 + + result = set_new_password(get_hashed_password(data["password"]), "local:" + current_user["email"]) + if result is not None: + # exception occurred + return dict(message="Password change failed. Please, try again and if the problem persists, contact NERD administrator."), 400 + else: + return dict(message="Password has been successfully changed!"), 200 + + + + +@api_v2.route('/refreshToken', methods=['POST']) +def refreshToken(): + """Resresh login token + --- + consumes: + - application/json + parameters: + - in: body + name: body + description: Request body for endpoint + required: true + schema: + type: object + properties: + token: + type: string + description: refresh token + example: "ey..." + responses: + 200: + description: Token refreshed + 400: + description: wrong format + 403: + description: Invalid Authentication token + 404: + description: Email address not verified + """ + data = request.json + try: + out = jwt.decode(data["token"], config.get("secret_key"), algorithms=["HS256"]) + except Exception as e: + return { + "message": "Something went wrong", + "data": None, + "error": "wrong format" + }, 400 + # check if anything changed in between token gens + current_user = get_user_by_id(out["user"]["id"]) + if current_user is None: + return { + "message": "Invalid Authentication token!", + "data": None, + "error": "Unauthorized" + }, 403 + if not "registered" in current_user["groups"]: + return { + "message": "Email address not verified", + "data": None, + }, 404 + out2 = out + out['exp'] = datetime.utcnow()+timedelta(hours=4) + out2['exp'] = datetime.utcnow()+timedelta(hours=72) + accessToken = jwt.encode(out, config.get('secret_key'), algorithm="HS256") + refreshToken = jwt.encode(out2, config.get('secret_key'), algorithm="HS256") + return Response(json.dumps([accessToken, refreshToken], default=str), 200, mimetype='application/json') + +######################################################## +# ADMIN ENDPOINTS # +######################################################## + +@api_v2.route('/users', methods=['GET']) +@token_required +def users_info(current_user): + """Gets all users + --- + security: + - OAuth2: [admin] + responses: + 200: + description: Users + 500: + description: Internal server error + """ + # check if user accessing this endpoint is admin + if 'admin' not in current_user["groups"]: + return dict(message='Access denied.'), 500 + + try: + users = get_users_admin() + except Exception as e: + return { + "message": "Something went wrong!", + "error": str(e), + "data": None + }, 500 + + return jsonify(users), 200 + + +@api_v2.route('/roles', methods=['PUT']) +@token_required +def change_roles(current_user): + """Change of user roles + --- + security: + - OAuth2: [admin] + consumes: + - application/json + parameters: + - in: body + name: body + description: Request body for endpoint + required: true + schema: + type: object + properties: + id: + type: string + description: user email + example: "local:user@email.com" + roles: + type: array + description: array of user roles + example: ["registered", "admin"] + responses: + 200: + description: Roles changed successfully + 500: + description: Internal server error + """ + # check if user accessing this endpoint is admin + if 'admin' not in current_user["groups"]: + return dict(message='Access denied.'), 500 + + data = request.json + # user whose roles are to be changed + ide = data["id"] + # new array of roles + roles = data["roles"] + try: + out = set_new_roles(ide, roles) + except Exception as e: + return { + "message": "Something went wrong!", + "error": str(e), + "data": None + }, 500 + if not out: + return { + "message": "Error while saving data to DB.", + "error": "DB Error", + "data": None + }, 500 + + return dict(message="Roles chaged successfully !"), 200 + +@api_v2.route('/delete_user/', methods=['DELETE']) +@token_required +def delete(current_user, ide): + """Deletes user + --- + security: + - OAuth2: [admin] + parameters: + - name: ide + in: path + type: string + required: true + description: ID of user to delete + responses: + 200: + description: User deleted successfully + 500: + description: Internal server error + """ + # check if user accessing this endpoint is admin + if 'admin' not in current_user["groups"]: + return dict(message='Access denied.'), 500 + + try: + delete_user(ide) + except Exception as e: + return { + "message": "Something went wrong!", + "error": str(e), + "data": None + }, 500 + + return dict(message="User deleted successfully!"), 200 + +@api_v2.route('/add-user', methods=['POST']) +@token_required +def add_user(current_user): + """Add new user + --- + security: + - OAuth2: [admin] + consumes: + - application/json + parameters: + - in: body + name: body + description: Request body for endpoint + required: true + schema: + type: object + properties: + email: + type: string + description: user email + example: "local:user@email.com" + password: + type: string + description: users passroed + example: "password123" + organization: + type: string + description: users organization + example: "org" + roles: + type: array + description: array of user roles + example: ["registered", "admin"] + verify: + type: boolean + description: mark email as verified + example: true + responses: + 200: + description: User added succesfully + 400: + description: Error while creating user + 500: + description: Internal server error + """ + # check if user accessing this endpoint is admin + if 'admin' not in current_user["groups"]: + return dict(message='Access denied.'), 500 + data = request.json + hashed_password = get_hashed_password(data["password"]) + res = create_user(data["email"], hashed_password, "local", data["name"], data["organization"], data["roles"]) + if isinstance(res, Exception): + if res.args[0].startswith("duplicate key") and "Key (id)" in res.args[0]: #TODO? neexistuje lep�� zp�sob kontroly typu cyhby? Pr� na to speci�ln� typ Exc asi nen� + return dict(message=f"User with email address {data['email']} already exists!"), 400 + else: + return dict(message=f"ERROR in create_user(): Something has failed during user creation process: {res.args}"), 400 + if data["verify"]: + just_verify_user_by_id("local:" + data["email"]) + else: + out = {} + out['user'] = { + 'email': data['email'], + } + out['exp'] = datetime.utcnow()+timedelta(hours=24) + encoded_jwt = jwt.encode(out, config.get('secret_key'), algorithm="HS256") + msg = Message(subject="[NERD] Account cerated", + recipients=[data["email"]], + reply_to=current_app.config.get('MAIL_DEFAULT_SENDER'), + ) + msg.html = f"

NERD

Network Entity Reputation Database

Email address verification

Dear user {data['email']},

you can activate your NERD account by clicking the link below (the link is valid for 24 hours):

{config.get('email_web')}/verify


NERD administrator

" + mailer.send(msg) + set_verification_email_sent(datetime.utcnow(), data["email"]) + + return dict(message=f"Account created for {data['email']}!"), 200 + + +######################################################## +# EXTERNAL IDENTITY PROVIDERS # +######################################################## + + +@api_v2.route('/oauth/google', methods=['GET']) +def oauth_google(): + code = request.args.get("code") + # Set the token endpoint URL + token_url = "https://oauth2.googleapis.com/token" + + # Set the parameters for the POST request + client_id = "864125509519-s4prljrk97usreg167i7de2rkppfa4re.apps.googleusercontent.com" + client_secret = "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" + redirect_uri = APP_BASE_URL + "/nerd/api/v2/oauth/google" + grant_type = "authorization_code" + + # Make the POST request to exchange the authorization code for an access token + response = requests.post(token_url, data={ + "code": code, + "client_id": client_id, + "client_secret": client_secret, + "redirect_uri": redirect_uri, + "grant_type": grant_type + }) + + # Get the access token and user info from the response + access_token = response.json()["access_token"] + token_type = response.json()["token_type"] + expires_in = response.json()["expires_in"] + refresh_token = response.json()["refresh_token"] + + # Use the access token to get user info + user_info_url = "https://www.googleapis.com/oauth2/v3/userinfo" + headers = {"Authorization": f"{token_type} {access_token}"} + user_info_response = requests.get(user_info_url, headers=headers) + + # Get the user info from the response + user_info = user_info_response.json() + + created = "false" + + # Now you can check if the user's email is in your local DB + if get_user_by_id("google:" + user_info["email"]) is None: + res = create_user(user_info["email"], None, "google", "NoName", "NoOrg") + if isinstance(res, Exception): + if res.args[0].startswith("duplicate key") and "Key (id)" in res.args[0]: #TODO? neexistuje lep�� zp�sob kontroly typu cyhby? Pr� na to speci�ln� typ Exc asi nen� + return dict(message=f"User with email address {data['email']} already exists! You can either log in or try to " + f"reset your password in log-in section."), 400 + else: + return dict(message=f"ERROR in register_user(): Something has failed during registration process: {res.args}"), 400 + else: + verify_user("google:" + user_info["email"]) + created = "true" + out = {} + out['user'] = { + 'login_type': 'google', + 'id': 'google:' + user_info["email"], + 'email': user_info["email"], + } + out['exp'] = datetime.utcnow()+timedelta(hours=4) + token = jwt.encode(out, config.get('secret_key'), algorithm="HS256") + out2 = {} + out2['user'] = { + 'login_type': 'google', + 'id': 'google:' + user_info["email"], + 'email': user_info["email"], + } + out2['exp'] = datetime.utcnow()+timedelta(hours=72) + refreshToken = jwt.encode(out2, config.get('secret_key'), algorithm="HS256") + set_last_login(datetime.utcnow(), 'google:' + user_info["email"]) + return redirect(APP_BASE_URL + "/nerd2/auth?token=" + token + "&refreshToken=" + refreshToken + "&created=" + created, code=302) + + +@api_v2.route('/oauth/twitter/url', methods=['GET']) +def oauth_twitter_access(): + #create an object of OAuth1Session + request_token = OAuth1Session(client_key="pOVnN86c7LgVcPhtnbc2dpmpu", client_secret="cRacpbtBs87wx3uEHQ6aE04SmyS0JpwlZG8W5TfL9WElMp7hVq") + # twitter endpoint to get request token + url = 'https://api.twitter.com/oauth/request_token' + # get request_token_key, request_token_secret and other details + data = request_token.get(url) + # split the string to get relevant data + data_token = str.split(data.text, '&') + ro_key = str.split(data_token[0], '=') + ro_secret = str.split(data_token[1], '=') + resource_owner_key = ro_key[1] + resource_owner_secret = ro_secret[1] + return redirect("https://api.twitter.com/oauth/authenticate?oauth_token=" + resource_owner_key) + +@api_v2.route('/oauth/twitter', methods=['GET']) +def oauth_twitter(): + token = request.args.get("oauth_token") + verifier = request.args.get("oauth_verifier") + oauth_token = OAuth1Session(client_key="pOVnN86c7LgVcPhtnbc2dpmpu", client_secret="cRacpbtBs87wx3uEHQ6aE04SmyS0JpwlZG8W5TfL9WElMp7hVq") + url = 'https://api.twitter.com/oauth/access_token' + data = {"oauth_verifier": verifier, "oauth_token": token} + + access_token_data = oauth_token.post(url, data=data) + access_token_list = str.split(access_token_data.text, '&') + + access_token_key = str.split(access_token_list[0], '=') + access_token_secret = str.split(access_token_list[1], '=') + access_token_name = str.split(access_token_list[3], '=') + access_token_id = str.split(access_token_list[2], '=') + key = access_token_key[1] + secret = access_token_secret[1] + oauth_user = OAuth1Session(client_key="pOVnN86c7LgVcPhtnbc2dpmpu", client_secret="cRacpbtBs87wx3uEHQ6aE04SmyS0JpwlZG8W5TfL9WElMp7hVq", + resource_owner_key=key, + resource_owner_secret=secret) + url_user = 'https://api.twitter.com/1.1/account/verify_credentials.json' + params = {"include_email": 'true'} + user_data = oauth_user.get(url_user, params=params) + user_info = user_data.json() + created = "false" + if get_user_by_id('twitter:' + user_info["email"]) is None: + res = create_user(user_info["email"], None, "twitter", "NoName", "NoOrg") + if isinstance(res, Exception): + if res.args[0].startswith("duplicate key") and "Key (id)" in res.args[0]: + return dict(message=f"User with email address {data['email']} already exists! You can either log in or try to " + f"reset your password in log-in section."), 400 + else: + return dict(message=f"ERROR in register_user(): Something has failed during registration process: {res.args}"), 400 + else: + verify_user("twitter:" + user_info["email"]) + created = "true" + out = {} + out['user'] = { + 'login_type': 'twitter', + 'id': 'twitter:' + user_info["email"], + 'email': user_info["email"], + } + out['exp'] = datetime.utcnow()+timedelta(hours=4) + token = jwt.encode(out, config.get('secret_key'), algorithm="HS256") + out2 = {} + out2['user'] = { + 'login_type': 'twitter', + 'id': 'twitter:' + user_info["email"], + 'email': user_info["email"], + } + out2['exp'] = datetime.utcnow()+timedelta(hours=72) + refreshToken = jwt.encode(out2, config.get('secret_key'), algorithm="HS256") + set_last_login(datetime.utcnow(), 'twitter:' + user_info["email"]) + return redirect(APP_BASE_URL + "/nerd2/auth?token=" + token + "&refreshToken=" + refreshToken + "&created=" + created, code=302) + +@api_v2.route('/oauth/github', methods=['GET']) +def oauth_github(): + code = request.args.get("code") + # Set the token endpoint URL + token_url = "https://github.com/login/oauth/access_token" + + # Set the parameters for the POST request + client_id = "b13789eb1250d5e77992" + client_secret = "c9de015e91e288f96973b7aa95406c44284687b4" + + # Make the POST request to exchange the authorization code for an access token + response = requests.post(token_url, data={ + "code": code, + "client_id": client_id, + "client_secret": client_secret + }) + + access_token = 'token ' + response.text.split("&")[0].split("=")[1] + url = 'https://api.github.com/user/emails' + headers = {"Authorization": access_token} + + resp = requests.get(url=url, headers=headers) + + userData = resp.json() + for email in userData: + if email["primary"]: + user_email = email["email"] + break + created = "false" + + if get_user_by_id('github:' + user_email) is None: + res = create_user(user_email, None, "github", "NoName", "NoOrg") + if isinstance(res, Exception): + if res.args[0].startswith("duplicate key") and "Key (id)" in res.args[0]: + return dict(message=f"User with email address {data['email']} already exists! You can either log in or try to " + f"reset your password in log-in section."), 400 + else: + return dict(message=f"ERROR in register_user(): Something has failed during registration process: {res.args}"), 400 + else: + verify_user("github:" + user_email) + created = "true" + out = {} + out['user'] = { + 'login_type': 'github', + 'id': 'github:' + user_email, + 'email': user_email, + } + out['exp'] = datetime.utcnow()+timedelta(hours=4) + token = jwt.encode(out, config.get('secret_key'), algorithm="HS256") + out2 = {} + out2['user'] = { + 'login_type': 'github', + 'id': 'github:' + user_email, + 'email': user_email, + } + out2['exp'] = datetime.utcnow()+timedelta(hours=72) + refreshToken = jwt.encode(out2, config.get('secret_key'), algorithm="HS256") + set_last_login(datetime.utcnow(), 'github:' + user_email) + return redirect(APP_BASE_URL + "/nerd2/auth?token=" + token + "&refreshToken=" + refreshToken + "&created=" + created, code=302) + + +def get_well_known_metadata(): + response = requests.get("https://login.cesnet.cz/oidc/.well-known/openid-configuration") + response.raise_for_status() + return response.json() + + +def get_oauth2_session(**kwargs): + oauth2_session = OAuth2Session("029639e5-09f4-461d-859a-7b06aa29d61e", + scope=["profile", "email", "openid"], + redirect_uri=APP_BASE_URL + "/nerd/api/v2/oauth/eduid", + **kwargs) + return oauth2_session + +@api_v2.route("/oauth/edugain/url") +def login_edugain(): + well_known_metadata = get_well_known_metadata() + oauth2_session = get_oauth2_session() + authorization_url, state = oauth2_session.authorization_url(well_known_metadata["authorization_endpoint"]) + session["oauth_state"] = state + return redirect(authorization_url) + +@api_v2.route("/oauth/eduid") +def edu_id_callback(): + well_known_metadata = get_well_known_metadata() + oauth2_session = get_oauth2_session(state=request.args["state"]) + oauth_token = oauth2_session.fetch_token(well_known_metadata["token_endpoint"], + client_secret="73743e93-9560-4d5b-983f-33911afce589cb6c980a-4baa-47e0-8317-c11f786b4254", + code=request.args["code"])["id_token"] + resp = oauth2_session.get(well_known_metadata["userinfo_endpoint"]) + data = resp.json() + + # if the user just created a profile + created = "false" + + # EduGain user not in DB yet + eduUser = get_user_by_id('edugain:' + data["email"]) + + if eduUser is None: + # check if user is in DB from old NERD system + shiUser = get_user_by_id('shibboleth:' + data["email"]) + if shiUser is None: + # new user create profile + res = create_user(data["email"], None, "edugain", "NoName", "NoOrg") + if isinstance(res, Exception): + if res.args[0].startswith("duplicate key") and "Key (id)" in res.args[0]: + return dict(message=f"User with email address {data['email']} already exists! You can either log in or try to " + f"reset your password in log-in section."), 400 + else: + return dict(message=f"ERROR in register_user(): Something has failed during registration process: {res.args}"), 400 + else: + verify_user("edugain:" + data["email"]) + created = "true" + else: + # user from old NERD merge profile + res = create_user(data["email"], None, "edugain", shiUser["name"], shiUser["org"]) + if isinstance(res, Exception): + if res.args[0].startswith("duplicate key") and "Key (id)" in res.args[0]: + return dict(message=f"User with email address {data['email']} already exists! You can either log in or try to " + f"reset your password in log-in section."), 400 + else: + return dict(message=f"ERROR in register_user(): Something has failed during registration process: {res.args}"), 400 + else: + verify_user("edugain:" + data["email"]) + set_api_v1_token("edugain:" + data["email"], shiUser["api_token"]) + delete_user('shibboleth:' + data["email"]) + created = "true" + + out = {} + out['user'] = { + 'login_type': 'edugain', + 'id': 'edugain:' + data["email"], + 'email': data["email"], + } + out['exp'] = datetime.utcnow()+timedelta(hours=4) + token = jwt.encode(out, config.get('secret_key'), algorithm="HS256") + out2 = {} + out2['user'] = { + 'login_type': 'edugain', + 'id': 'edugain:' + data["email"], + 'email': data["email"], + } + out2['exp'] = datetime.utcnow()+timedelta(hours=72) + refreshToken = jwt.encode(out2, config.get('secret_key'), algorithm="HS256") + set_last_login(datetime.utcnow(), 'edugain:' + data["email"]) + return redirect(APP_BASE_URL + "/nerd2/auth?token=" + token + "&refreshToken=" + refreshToken + "&created=" + created, code=302) \ No newline at end of file diff --git a/NERDweb/api_v2_swag/login.yml b/NERDweb/api_v2_swag/login.yml new file mode 100644 index 00000000..579aa574 --- /dev/null +++ b/NERDweb/api_v2_swag/login.yml @@ -0,0 +1,29 @@ +User local login + --- + consumes: + - application/json + parameters: + - in: body + name: body + description: Request body for endpoint + required: true + schema: + type: object + properties: + email: + type: string + description: Email of the local user + example: "user@rmail.com" + password: + type: string + description: Password of the local user + example: "password123" + responses: + 200: + description: Login successful + 400: + description: Bad password + 404: + description: User does not exist + 500: + description: Inetrnal server error \ No newline at end of file diff --git a/NERDweb/api_v2_swag/register.yml b/NERDweb/api_v2_swag/register.yml new file mode 100644 index 00000000..1b1164c0 --- /dev/null +++ b/NERDweb/api_v2_swag/register.yml @@ -0,0 +1,37 @@ +User local registration + --- + consumes: + - application/json + parameters: + - in: body + name: body + description: Request body for endpoint + required: true + schema: + type: object + properties: + email: + type: string + description: Email of the local user + example: "user@rmail.com" + password: + type: string + description: Password of the local user + example: "password123" + organization: + type: string + enum: + - Academic + - Research + - Security + description: Organization of the local user + example: "user@rmail.com" + name: + type: string + description: Name of the local user + example: "noName for v2" + responses: + 200: + description: Registration successful + 400: + description: Error during registration \ No newline at end of file diff --git a/NERDweb/api_v2_swag/search.yml b/NERDweb/api_v2_swag/search.yml new file mode 100644 index 00000000..3a8ce5e5 --- /dev/null +++ b/NERDweb/api_v2_swag/search.yml @@ -0,0 +1,161 @@ +Basic info for a single IP address + --- + consumes: + - application/json + parameters: + - in: body + name: body + description: Request body for endpoint + required: true + schema: + type: object + properties: + subnet: + nullable: true + type: ['null', array] + items: + type: string + description: IPv4 prefix/subnet in CIDR format + example: ["146.88.240.4", "192.168.1.0/24"] + hostname: + nullable: true + type: ['null', array] + items: + type: string + description: Hostname suffix + example: ["security.criminalip.com"] + country: + nullable: true + type: ['null', array] + items: + type: string + description: IP address country of origin + example: ["US"] + asn: + nullable: true + type: ['null', array] + description: Autonomous system number + example: AS202425 + cat: + nullable: true + type: ['null', array] + description: Event category - Warden + example: AnomalyBehaviour + blacklist: + nullable: true + type: ['null', array] + description: Appearance on blacklist(s) + example: AbuseIPDB + source: + nullable: true + type: ['null', array] + description: IP address malicious intent detected by source + example: warden + tag: + nullable: true + type: ['null', array] + description: Additional tags + example: attemptexploit + sort: + nullable: true + type: ['null', string] + enum: + - ip + - ts_added + - last_activity + - rep + description: allowed - ip, rep, ts_added and last_activity + example: ip + order: + nullable: true + type: ['null', string] + enum: + - asc + - desc + description: Order ascending or descending + example: desc + limit: + nullable: true + type: ['null', number] + enum: + - 5 + - 10 + - 20 + - 50 + description: results per page + example: 20 + asn_op: + nullable: true + type: ['null', string] + enum: + - AND + - OR + description: AND / OR logical operator that parses through given asn values + example: or + source_op: + nullable: true + type: ['null', string] + enum: + - AND + - OR + description: AND / OR logical operator that parses through given source values + example: or + cat_op: + nullable: true + type: ['null', string] + enum: + - AND + - OR + description: AND / OR logical operator that parses through given category values + example: or + bl_op: + nullable: true + type: ['null', string] + enum: + - AND + - OR + description: AND / OR logical operator that parses through given blacklist values + example: or + tag_op: + nullable: true + type: ['null', string] + enum: + - AND + - OR + description: AND / OR logical operator that parses through given tag values + example: or + whitelisted: + nullable: true + type: boolean + description: Hide or show whitelisted IPs + example: true + page: + nullable: true + type: number + description: Page number when listing muliple results + example: 1 + responses: + '200': + description: OK + schema: + type: object + properties: + message: + type: string + example: Endpoint response message + '400': + description: Bad Request + schema: + type: object + properties: + message: + type: string + example: Endpoint response message + '500': + description: Internal Server Error + schema: + type: object + properties: + message: + type: string + example: Endpoint response message \ No newline at end of file diff --git a/NERDweb/auth.py b/NERDweb/auth.py new file mode 100644 index 00000000..00570bb1 --- /dev/null +++ b/NERDweb/auth.py @@ -0,0 +1,51 @@ +from functools import wraps +import jwt +from flask import request, abort +from flask import current_app +from userdb import get_user_by_id +import os +import common.config + + +config = common.config.read_config(os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "/etc/nerd/nerdweb.yml"))) + +def token_required(f): + @wraps(f) + def decorated(*args, **kwargs): + token = None + if "Authorization" in request.headers: + token = request.headers["Authorization"] + if not token: + return { + "message": "Authentication Token is missing!", + "data": None, + "error": "Unauthorized" + }, 401 + try: + data=jwt.decode(token, config.get("secret_key"), algorithms=["HS256"]) + + if data["user"]["login_type"] == "devel": + return data + current_user=get_user_by_id(data["user"]["id"]) + if current_user is None: + return { + "message": "Invalid Authentication token!", + "data": None, + "error": "Unauthorized" + }, 401 + if not "registered" in current_user["groups"]: + return { + "message": "Email address not verified", + "data": None, + }, 403 + except Exception as e: + return { + "message": "Something went wrong", + "data": str(e), + "error": "AuthToken wrong" + }, 401 + + return f(current_user, *args, **kwargs) + + return decorated + diff --git a/NERDweb/nerd_main.py b/NERDweb/nerd_main.py index e9311ac6..c3f70661 100644 --- a/NERDweb/nerd_main.py +++ b/NERDweb/nerd_main.py @@ -26,6 +26,8 @@ from pymisp import ExpandedPyMISP from ipaddress import IPv4Address, AddressValueError from event_count_logger import EventCountLogger, EventGroup, DummyEventGroup +# Open API requirements +from flasgger import Swagger # Add to path the "one directory above the current file location" sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))) @@ -2319,6 +2321,256 @@ def get_shodan_response(ipaddr=None): # ********** +# ***************************** API v2 helper functions ******************************************** +def create_query_v2(data): + # Prepare 'find' part of the query + queries = [] + if data is None: + return None + + if "subnet" in data and data["subnet"] is not None and len(data["subnet"]) != 0: + subqueries = [] + for subnet in data["subnet"]: + subnet = ipaddress.IPv4Network(subnet, strict=False) + subnet_start = int(subnet.network_address) # IP addresses are stored as int + subnet_end = int(subnet.broadcast_address) + subqueries.append( {'$and': [{'_id': {'$gte': subnet_start}}, {'_id': {'$lte': subnet_end}}]} ) + queries.append({'$or': subqueries}) + + if "hostname" in data and data["hostname"] is not None and len(data["hostname"]) != 0: + subqueries = [] + for hostname in data["hostname"]: + hn = hostname[::-1] # Hostnames are stored reversed in DB to allow search by suffix as a range search + hn_end = hn[:-1] + chr(ord(hn[-1])+1) + subqueries.append( {'$and': [{'hostname': {'$gte': hn}}, {'hostname': {'$lt': hn_end}}]} ) + queries.append({'$or': subqueries}) + + if "country" in data and data["country"] is not None and len(data["country"]) != 0: + queries.append( { '$or': [{'geo.ctry': c.upper() } for c in data["country"]]} ) + + if "asn" in data and data["asn"] is not None and len(data["asn"]) != 0 : + subqueries = [] + for asn in data["asn"]: + # ASN is not stored in IP records - get list of BGP prefixes of the ASN and filter by these + asn = int(asn.lstrip("ASas")) + asrec = mongo.db.asn.find_one({'_id': asn}) + if asrec and 'bgppref' in asrec: + subqueries.append( {'bgppref': {'$in': asrec['bgppref']}} ) + else: + subqueries.append( {'_id': {'$exists': False}} ) # ASN not in DB, add query which is always false to get no results + op = '$and' if (data["asn_op"] == "AND") else '$or' + queries.append({op: subqueries}) + + if "source" in data and data["source"] is not None and len(data["source"]) != 0: + op = '$and' if (data["source_op"] == "AND") else '$or' + queries.append( {op: [{'_ttl.' + s.lower(): {'$exists': True}} for s in data["source"]]} ) + + if "cat" in data and data["cat"] is not None and len(data["cat"]) != 0: + op = '$and' if (data["cat_op"] == "AND") else '$or' + queries.append( {op: [{'events.cat': cat} for cat in data["cat"]]} ) + + if "node" in data and data["node"] is not None and len(data["node"]) != 0: + op = '$and' if (data["node_op"] == "AND") else '$or' + queries.append( {op: [{'events.node': node} for node in data["node"]]} ) + + if "blacklist" in data and data["blacklist"] is not None and len(data["blacklist"]) != 0: + op = '$and' if (data["bl_op"] == "AND") else '$or' + array = [{('dbl' if t == 'd' else 'bl'): {'$elemMatch': {'n': id, 'v': 1}}} for t,_,id in map(lambda s: s.partition(':'), data["blacklist"])] + queries.append( {op: array} ) + + if "tag" in data and data["tag"] is not None and len(data["tag"]) != 0: + op = '$and' if (data["tag_op"] == "AND") else '$or' + queries.append( {op: [{'tags.'+ tag_id: {'$exists': True}} for tag_id in data["tag"]]} ) + + if "whitelisted" in data and data["whitelisted"]: + queries.append( {'tags.whitelist': {'$exists': False}} ) + + query = {'$and': queries} if queries else None + return query + +def get_basic_info_dic_v2(val): + geo_d = {} + if 'geo' in val.keys(): + geo_d['ctry'] = val['geo'].get('ctry', "unknown") + + bl_l = [] + for l in val.get('bl', []): + bl_l.append(l['n']) # TODO: shouldn't there be a check for v=1? + + tags_l = [] + for l in val.get('tags', []): + d = { + 'n' : l, + 'c' : val['tags'][l]['confidence'] + } + + tags_l.append(d) + + data = { + 'ip' : val['_id'], + 'rep' : val.get('rep', 0.0), + 'fmp' : val.get('fmp', {'general': 0.0}), + 'hostname' : (val.get('hostname', '') or '')[::-1], + 'ipblock' : val.get('ipblock', ''), + 'bgppref' : val.get('bgppref', ''), + 'asn' : val.get('asn',[]), + 'geo' : geo_d, + 'bl' : bl_l, + 'tags' : tags_l, + 'ts_last_update' : val.get('ts_last_update', ''), + 'ts_added' : val.get('ts_added', ''), + } + + return data + +def find_ip_data(query, skip_n, limit): + return mongo.db.ip.find(query).skip(skip_n).limit(limit) + +def ip_to_warden_data(ipaddr): + ipint = ipstr2int(ipaddr) + ipinfo = mongo.db.ip.aggregate(pipeline = [ + { '$match': { '_id': ipint } }, + { '$unwind': '$events' }, + { '$group': { + '_id': { + 'date': '$events.date', + 'cat': '$events.cat' + }, + 'n_sum': { '$sum': '$events.n' }, + 'conns_sum': { '$sum': '$events.conns' }, + } + }, + { '$group': { + '_id': '$_id.date', + 'categories': { + "$push": { + "k": "$_id.cat", + "v": { + "n_sum": "$n_sum", + "conns_sum": "$conns_sum", + "nodes": "$nodes" + } + } + } + } + }, + { '$project': { + '_id': 0, + 'date': '$_id', + "categories": {"$arrayToObject": "$categories"} + } + }, + { '$sort': { 'date': 1 } } + ]) + + ipinfo_list = [doc for doc in ipinfo] + #ipinfo_list['_id'] = int2ipstr(ipinfo_list['_id']) + return ipinfo_list + + +# additional data for selected users that belong to group +def ip_to_warden_data_with_nodes(ipaddr): + ipint = ipstr2int(ipaddr) + ipinfo = mongo.db.ip.aggregate(pipeline = [ + { '$match': { '_id': 1246899993 } }, + { '$unwind': '$events' }, + { '$group': { + '_id': { + 'date': '$events.date', + 'cat': '$events.cat', + 'node': '$events.node' + }, + 'n_sum': { '$sum': '$events.n' }, + 'conns_sum': { '$sum': '$events.conns' } + }}, + { '$group': { + '_id': { + 'date': '$_id.date', + 'cat': '$_id.cat' + }, + 'n_sum': { '$sum': '$n_sum' }, + 'conns_sum': { '$sum': '$conns_sum' }, + 'nodes': { + '$push': { + 'node': '$_id.node', + 'n_sum': '$n_sum', + 'conns_sum': '$conns_sum' + } + } + }}, + { '$group': { + '_id': '$_id.date', + 'categories': { + '$push': { + 'cat': '$_id.cat', + 'n_sum': '$n_sum', + 'conns_sum': '$conns_sum', + 'nodes': '$nodes' + } + } + }}, + { '$project': { + '_id': 0, + 'date': '$_id', + 'categories': 1 + }}, + { '$sort': { 'date': -1 } } + ]) + + ipinfo_list = [doc for doc in ipinfo] + #ipinfo_list['_id'] = int2ipstr(ipinfo_list['_id']) + return ipinfo_list + + +# ***************************************************************************************** + +swagger_config = { + "headers": [ + ], + "specs": [ + { + "endpoint": 'apispec_1', + "route": '/apispec_1.json', + "rule_filter": lambda rule: True, # all in + "model_filter": lambda tag: True, # all in + } + ], + "static_url_path": "/flasgger_static", + # "static_folder": "static", # must be set by user + "swagger_ui": True, + "specs_route": "/apidocs/" +} +template = { + "swagger": "2.0", + "info": { + "title": "NERD API v2", + "description": "Second API version for new NERD system, built to interact with Vue.js FE", + "contact": { + "responsibleOrganization": "Liberouter", + "responsibleDeveloper": "NERD", + "email": "bartos@cesnet.cz", + "url": "https://nerd.cesnet.cz/", + }, + }, + "host": "127.0.0.1", # overrides localhost:500 + "basePath": "/doc", # base bash for blueprint registration + "schemes": [ + "http", + "https" + ], + "operationId": "getmyData" +} + +app.config['SWAGGER_VALIDATOR_ENABLE'] = True +swagger = Swagger(app, config=swagger_config, template=template, parse=True) + +# register blueprints - generally definitions of routes in separate files +from user_management import user_management +from api_v2 import api_v2 + +app.register_blueprint(user_management, url_prefix="/user") +app.register_blueprint(api_v2, url_prefix="/api/v2") + if __name__ == "__main__": # Set global testing flag config.testing = True diff --git a/NERDweb/user_management.py b/NERDweb/user_management.py new file mode 100644 index 00000000..4db27511 --- /dev/null +++ b/NERDweb/user_management.py @@ -0,0 +1,127 @@ +from datetime import datetime, timedelta + +from flask import Blueprint, flash, redirect, session, g, make_response, current_app, url_for, request, jsonify +from flask_wtf import FlaskForm +from wtforms import StringField, PasswordField, SubmitField +from wtforms.validators import DataRequired, Email, EqualTo, Length, Regexp +from bcrypt import hashpw, checkpw, gensalt +from flask_mail import Message +from itsdangerous import URLSafeTimedSerializer, BadSignature, SignatureExpired +from flask_dance.consumer import OAuth2ConsumerBlueprint +import jwt + + +# needed overridden render_template method because it passes some needed attributes to Jinja templates +from nerd_main import render_template, BASE_URL, config, mailer +from userdb import create_user, get_user_data_for_login, get_user_by_email, verify_user, get_verification_email_sent, \ + set_verification_email_sent, set_last_login, get_user_name, set_new_password, check_if_user_exists, get_user_by_id + +ERROR_MSG_MISSING_MAIL_CONFIG = "ERROR: No destination email address configured. This is a server configuration " \ + "error. Please, report this to NERD administrator if possible." + +# variable name and name of blueprint is recommended to be same as filename +user_management = Blueprint("user_management", __name__, static_folder="static", template_folder="templates") + + +# ***** Util functions ***** +def generate_token(user_email): + """ Generates random token for email verification or password reset. """ + serializer = URLSafeTimedSerializer(current_app.config['SECRET_KEY']) + return serializer.dumps(user_email) + +def generate_jwt_token(email): + out = {} + out['user'] = { + 'login_type': 'local', + 'id': 'local:' + email, + 'email': email, + } + out['exp'] = datetime.utcnow()+timedelta(hours=4) + return jwt.encode(out, config.get('secret_key'), algorithm="HS256") + +def confirm_jwt_token(token): + try: + data=jwt.decode(token, config.get("secret_key"), algorithms=["HS256"]) + + if data["user"]["login_type"] == "devel": + return data + current_user=get_user_by_id(data["user"]["id"]) + if current_user is None: + return { + "message": "Invalid Authentication token!", + "data": data["user"]["id"], + "error": "Unauthorized" + }, 401 + if not "registered" in current_user["groups"]: + return { + "message": "Email address not verified", + "data": None, + }, 403 + except Exception as e: + return { + "message": "Something went wrong", + "data": str(e), + "error": "AuthToken wrong" + }, 401 + return current_user + + +def confirm_token(token, expiration=3600): + serializer = URLSafeTimedSerializer(current_app.config['SECRET_KEY']) + try: + email = serializer.loads( + token, + max_age=expiration + ) + except (BadSignature, SignatureExpired): + return False + return email + + +def get_current_datetime(): + return datetime.now() # TODO? shouldn't it be utcnow()? uklad�d� se do DB + + +def send_verification_email(user_email, name): + # TODO: udelat nekde kontrolu, ze je nakonfigurovane mailovani + token = generate_token(user_email) + confirm_url = url_for('user_management.verify_email', token=token, _external=True) + msg = Message(subject="[NERD] Verify your account email address", + recipients=[user_email], + reply_to=current_app.config.get('MAIL_DEFAULT_SENDER'), + body=f"Dear {name},\n\nplease verify your email address to complete your registration process by" + f" clicking on this link:\n\n{confirm_url}\n\nThank you,\nNERD administrator", + ) + mailer.send(msg) + + +def send_password_reset_email(user_email, name): + if not config.get('login.request-email', None): + return make_response(ERROR_MSG_MISSING_MAIL_CONFIG) + token = generate_token(user_email) + password_reset_url = url_for('user_management.password_reset', token=token, _external=True) + msg = Message(subject="[NERD] Password reset request", + recipients=[user_email], + reply_to=current_app.config.get('MAIL_DEFAULT_SENDER'), + body=f"Dear {name},\n\nyou can reset your password by clicking on this link:" + f"\n\n{password_reset_url}\n\nThank you,\nNERD administrator", + ) + mailer.send(msg) + + +def get_hashed_password(password): + return hashpw(password.encode('utf-8'), gensalt()).decode('utf-8') + + +def verify_email_token(token): + """ Verifies email token and returns user's email, to which the token was crafted and also user info. """ + email = confirm_token(token) + if not email: + flash('The link is invalid or has expired.', 'error') + return None + user = get_user_by_email(email) + if user is None: + flash('Such user account does not exist. Please check your link and if the problem persists, ' + 'contact NERD administrator.', 'error') + return user + diff --git a/NERDweb/userdb.py b/NERDweb/userdb.py index 1af598d2..0fd0fca3 100644 --- a/NERDweb/userdb.py +++ b/NERDweb/userdb.py @@ -51,7 +51,6 @@ def get_all_groups(): groups.discard('*') return sorted(list(groups)) - # ***** Access control functions ***** def get_user_groups(full_id): @@ -77,9 +76,6 @@ def ac(resource): return False return ac - -# TODO - split authentication and authorization/get_user_information - def get_user_info(session): """ Returun info about current user (or None if noone is logged in) and @@ -167,3 +163,155 @@ def generate_unique_token(user): return False return True + +############################################### +# NEW FUNCTIONS API v2 # +############################################### +# ***** User management functions ***** +def create_user(email, password, provider, name=None, organization=None, groups=[]): + try: + cur = db.cursor() + cur.execute("""INSERT INTO users (id, groups, name, email, org, password) + VALUES (%s, %s, %s, %s, %s, %s)""", + (provider + ":" + email, groups, name, email, organization, password)) + db.commit() + cur.close() + except (Exception, psycopg2.DatabaseError) as e: + return e + +def get_user_by_email(email): + cur = db.cursor() + cur.execute("SELECT * FROM users WHERE email=%s", (email,)) + row = cur.fetchone() + if not row: + return None + col_names = [col.name for col in cur.description] + return dict(zip(col_names, row)) + +def get_user_by_id(id): + cur = db.cursor() + cur.execute("SELECT * FROM users WHERE id=%s", (id,)) + row = cur.fetchone() + if not row: + return None + col_names = [col.name for col in cur.description] + return dict(zip(col_names, row)) + +def get_user_data_for_login(user_id): + cur = db.cursor() + cur.execute("SELECT id, password, name FROM users WHERE id = %s", (user_id,)) + row = cur.fetchone() + if not row: + return None + return {'id': row[0], 'password': row[1], 'name': row[2]} + +def check_if_user_exists(user_id): + cur = db.cursor() + cur.execute("SELECT id, name FROM users WHERE id = %s", (user_id,)) + row = cur.fetchone() + if not row: + return False + return True + +def verify_user(user_id): + try: + cur = db.cursor() + cur.execute("""UPDATE users SET groups=%s, verified=TRUE WHERE id = %s""", (["registered"], user_id,)) + db.commit() + cur.close() + except psycopg2.Error as e: + print(f"verify_user() failed: {e.pgerror}") + return e + +def verify_user_by_mail(mail): + try: + cur = db.cursor() + cur.execute("""UPDATE users SET groups=%s, verified=TRUE WHERE email = %s""", (["registered"], mail,)) + db.commit() + cur.close() + except psycopg2.Error as e: + print(f"verify_user() failed: {e.pgerror}") + return e + +def just_verify_user_by_id(ide): + try: + cur = db.cursor() + cur.execute("""UPDATE users SET verified=TRUE WHERE id = %s""", (ide,)) + db.commit() + cur.close() + except psycopg2.Error as e: + print(f"verify_user() failed: {e.pgerror}") + return e + +def set_verification_email_sent(date_time, email): + try: + cur = db.cursor() + cur.execute("""UPDATE users SET verification_email_sent=%s WHERE id = %s""", (date_time, id)) + db.commit() + cur.close() + except psycopg2.Error as e: + print(f"set_verification_email_sent() failed: {e.pgerror}") + return e + + +def set_last_login(date_time, ide): + try: + cur = db.cursor() + cur.execute("""UPDATE users SET last_login=%s WHERE id = %s""", (date_time, ide)) + db.commit() + cur.close() + except psycopg2.Error as e: + print(f"set_last_login() failed: {e.pgerror}") + return e + + +def get_verification_email_sent(user_email): + cur = db.cursor() + cur.execute("SELECT verification_email_sent FROM users WHERE email = %s", (user_email,)) + row = cur.fetchone() + if not row: + return None + return row[0] + + +def get_user_name(user_email): + cur = db.cursor() + cur.execute("SELECT name FROM users WHERE email = %s", (user_email,)) + row = cur.fetchone() + if not row: + return None + return row[0] + +def set_new_password(new_password, id): + try: + cur = db.cursor() + cur.execute("""UPDATE users SET password=%s WHERE id = %s""", (new_password, id)) + db.commit() + cur.close() + except psycopg2.Error as e: + print(f"set_new_password() failed: {e.pgerror}") + return e + + +def set_api_v1_token(ide, token): + cur = db.cursor() + cur.execute("UPDATE users SET api_token = %s WHERE id = %s", (token, ide,)) + return True + +def get_users_admin(): + cur = db.cursor() + cur.execute("SELECT email, groups, id, org, api_token, verified, verification_email_sent, last_login FROM users ORDER BY email") + row = cur.fetchall() + if not row: + return None + return row + +def set_new_roles(ide, roles): + cur = db.cursor() + cur.execute("UPDATE users SET groups = %s WHERE id = %s", (roles, ide,)) + return True + +def delete_user(ide): + cur = db.cursor() + cur.execute("DELETE FROM users WHERE id = %s", (ide,)) + return True \ No newline at end of file From 3a2a4f793f18e62e6927b7cc943bafe13abd8979 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krist=C3=ADna=20Oltmanov=C3=A1?= Date: Wed, 31 May 2023 15:18:55 +0200 Subject: [PATCH 2/2] removing circular dependencies nerd_main functions moved to com --- NERDweb/api_v2.py | 5 +- NERDweb/com.py | 407 ++++++++++++++++++++++++ NERDweb/nerd_main.py | 627 ++++++++++++++++--------------------- NERDweb/user_management.py | 21 +- 4 files changed, 706 insertions(+), 354 deletions(-) create mode 100644 NERDweb/com.py diff --git a/NERDweb/api_v2.py b/NERDweb/api_v2.py index 380774f4..8e343d58 100644 --- a/NERDweb/api_v2.py +++ b/NERDweb/api_v2.py @@ -22,9 +22,8 @@ from user_management import get_hashed_password, generate_token, verify_email_token, \ generate_jwt_token, confirm_jwt_token -from nerd_main import get_ip_info, get_basic_info_dic, create_query_v2, \ - find_ip_data, get_basic_info_dic_v2, attach_whois_data, mailer, mongo, \ - get_ip_blacklists, ip_to_warden_data + +from com import attach_whois_data, get_ip_info, mailer, mongo, get_basic_info_dic, get_basic_info_dic_short, get_basic_info_dic_v2, find_ip_data, ip_to_warden_data, create_query_v2 from userdb import get_user_info, authenticate_with_token, generate_unique_token, \ get_user_by_id, create_user, verify_user, set_last_login, get_user_name, \ set_new_password, get_users_admin, set_verification_email_sent, set_new_roles, \ diff --git a/NERDweb/com.py b/NERDweb/com.py new file mode 100644 index 00000000..7f9a3ca1 --- /dev/null +++ b/NERDweb/com.py @@ -0,0 +1,407 @@ +import os +import sys +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))) +import common.config +from flask_mail import Mail +from flask import Flask, Response +from flask_pymongo import PyMongo +from event_count_logger import EventCountLogger, EventGroup, DummyEventGroup +from flask_wtf import FlaskForm +from wtforms import validators, StringField, TextAreaField, FloatField, IntegerField, BooleanField, HiddenField, SelectField, SelectMultipleField, PasswordField +from common.utils import ipstr2int, int2ipstr, parse_rfc_time +from datetime import datetime, timedelta, timezone + +DEFAULT_CONFIG_FILE = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "/etc/nerd/nerdweb.yml")) + +# TODO parse arguments using ArgParse +if len(sys.argv) >= 2: + cfg_file = sys.argv[1] +else: + cfg_file = DEFAULT_CONFIG_FILE +cfg_dir = os.path.dirname(os.path.abspath(cfg_file)) + +# Read web-specific config (nerdweb.cfg) +config = common.config.read_config(cfg_file) + +# Read EventCountLogger config (to separate dict) and initialize loggers +ecl_cfg_filename = config.get('event_logging_config', None) +if ecl_cfg_filename: + # Load config + config_ecl = common.config.read_config(os.path.join(cfg_dir, ecl_cfg_filename)) + # Initialize EventCountLogger + ecl = EventCountLogger(config_ecl.get('groups'), config_ecl.get('redis', {})) + # Get instances of EventGroups (if specified in configuration, otherwise, DummyEventGroup is used, so logging is no-op) + # (it's recommended to enable local counters for both groups for better performance) + log_ep = ecl.get_group('web_endpoints') or DummyEventGroup() # log access to individual endpoints + log_err = ecl.get_group('web_errors') or DummyEventGroup() # log error replies +else: + print("WARNING: nerd_main: Path to event logging config ('event_logging_config' key) not specified, EventCountLogger disabled.") + log_ep = DummyEventGroup() + log_err = DummyEventGroup() + +app = Flask(__name__) + +app.secret_key = config.get('secret_key') + +app.jinja_env.trim_blocks = True +app.jinja_env.lstrip_blocks = True + +# Configuration of PyMongo +mongo_dbname = config.get('mongodb.dbname', 'nerd') +mongo_host = config.get('mongodb.host', 'localhost:27017') +if isinstance(mongo_host, list): + mongo_host = ','.join(mongo_host) +mongo_uri = "mongodb://{}/{}".format(mongo_host, mongo_dbname) +mongo_rs = config.get('mongodb.rs', None) +if mongo_rs: + mongo_uri += '?replicaSet='+mongo_rs +app.config['MONGO_URI'] = mongo_uri +print("MongoDB: Connecting to: {}".format(mongo_uri)) + +mongo = PyMongo(app) + +# Configuration of MAIL extension +app.config['MAIL_SERVER'] = config.get('mail.server', 'localhost') +app.config['MAIL_PORT'] = config.get('mail.port', '25') +app.config['MAIL_USE_TLS'] = config.get('mail.tls', False) +app.config['MAIL_USE_SSL'] = config.get('mail.ssl', False) +app.config['MAIL_USERNAME'] = config.get('mail.username', None) +app.config['MAIL_PASSWORD'] = config.get('mail.password', None) +app.config['MAIL_DEFAULT_SENDER'] = config.get('mail.sender', 'NERD ') + +mailer = Mail(app) + +def validator_optional(form, field): + if not field.data: + field.errors[:] = [] + raise validators.StopValidation() + +def strip_whitespace(s): + if isinstance(s, str): + s = s.strip() + return s + + +def clean_secret_data(data): + """Remove all keys starting with '_' (except '_id') from dict.""" + if data is not None: + for i in list(data): + if i.startswith("_") and i != "_id": + del data[i] + return data + +def attach_whois_data(ipinfo, full): + """ + Attach records of related entities to given IP record. + + If full==True, attach full records of BGP prefix, ASNs, IP block, Org entities (as 'bgppref, 'asn', 'ipblock' and 'org' keys), + otherwise only attach list of ASN numbers (as 'asn' key). + """ + if not full: + # Only attach ASN number(s) + if 'bgppref' in ipinfo: + bgppref_rec = mongo.db.bgppref.find_one({'_id': ipinfo['bgppref']}, {'asn': 1}) + if bgppref_rec is None: + print("ERROR: Can't find BGP prefix '{}' in database (trying to enrich IP {})".format(ipinfo['bgppref'], ipinfo['_id'])) + return + if 'asn' in bgppref_rec: + ipinfo['asn'] = bgppref_rec['asn'] + return + + # Full - attach full records of related BGP prefix, ASNs, IP block, Org + # IP->BGPpref + if 'bgppref' in ipinfo: + bgppref_rec = clean_secret_data(mongo.db.bgppref.find_one({'_id': ipinfo['bgppref']})) + if bgppref_rec is None: + print("ERROR: Can't find BGP prefix '{}' in database (trying to enrich IP {})".format(ipinfo['bgppref'], ipinfo['_id'])) + else: + # BGPpref->ASN(s) + asn_list = [] + for asn in bgppref_rec['asn']: + asn_rec = clean_secret_data(mongo.db.asn.find_one({'_id': asn})) + if asn_rec is None: + print("ERROR: Can't find ASN '{}' in database (trying to enrich IP {}, bgppref {})".format(asn, ipinfo['_id'], bgppref_rec['_id'])) + else: + # ASN->Org + if 'org' in asn_rec: + org_rec = clean_secret_data(mongo.db.org.find_one({'_id': asn_rec['org']})) + if org_rec is None: + print( + "ERROR: Can't find Org '{}' in database (trying to enrich IP {}, bgppref {}, ASN {})".format( + asn_rec['org'], ipinfo['_id'], bgppref_rec['_id'], asn)) + else: + conv_dates(org_rec) + asn_rec['org'] = org_rec + + del asn_rec['bgppref'] + conv_dates(asn_rec) + asn_list.append(asn_rec) + + del bgppref_rec['asn'] + conv_dates(bgppref_rec) + ipinfo['bgppref'] = bgppref_rec + ipinfo['asn'] = asn_list + + # IP->ipblock + if 'ipblock' in ipinfo: + ipblock_rec = (mongo.db.ipblock.find_one({'_id': ipinfo['ipblock']})) + if ipblock_rec is None: + print("ERROR: Can't find IP block '{}' in database (trying to enrich IP {})".format(ipinfo['ipblock'], + ipinfo['_id'])) + else: + # ipblock->org + if "org" in ipblock_rec: + org_rec = clean_secret_data(mongo.db.org.find_one({'_id': ipblock_rec['org']})) + if org_rec is None: + print("ERROR: Can't find Org '{}' in database (trying to enrich IP {}, ipblock '{}')".format( + ipblock_rec['org'], ipinfo['_id'], ipblock_rec['_id'])) + else: + conv_dates(org_rec) + ipblock_rec['org'] = org_rec + + conv_dates(ipblock_rec) + ipinfo['ipblock'] = ipblock_rec + +def conv_dates(rec): + """Convert datetimes in a record to YYYY-MM-DDTMM:HH:SS string""" + for key in ('ts_added', 'ts_last_update'): + if key in rec and isinstance(rec[key], datetime): + rec[key] = rec[key].strftime("%Y-%m-%dT%H:%M:%S") + +class SingleIPForm(FlaskForm): + ip = StringField('IP address', [validator_optional, validators.IPAddress(message="Invalid IPv4 address")], filters=[strip_whitespace]) + +def get_ip_info(ipaddr, full): + data = { + 'err_n': 400, + 'error': "No IP address specified", + 'ip': ipaddr + } + + if not ipaddr: + log_err.log('400_bad_request') + return False, Response(json.dumps(data), 400, mimetype='application/json') + + form = SingleIPForm(ip=ipaddr) + if not form.validate(): + log_err.log('400_bad_request') + data['error'] = "Bad IP address" + return False, Response(json.dumps(data), 400, mimetype='application/json') + + ipint = ipstr2int(form.ip.data) # Convert string IP to int + + if full: + ipinfo = mongo.db.ip.find_one({'_id': ipint}) + else: + ipinfo = mongo.db.ip.find_one({'_id': ipint}, + {'rep': 1, 'fmp': 1, 'hostname': 1, 'bgppref': 1, 'ipblock': 1, 'geo': 1, 'bl': 1, + 'tags': 1}) + if not ipinfo: + log_err.log('404_api_ip_not_found') + data['err_n'] = 404 + data['error'] = "IP address not found" + return False, Response(json.dumps(data), 404, mimetype='application/json') + + ipinfo['_id'] = int2ipstr(ipinfo['_id']) # Convert int IP to string + + attach_whois_data(ipinfo, full) + return True, ipinfo + +# ***** NERD API BasicInfo - helper funcs ***** +def get_basic_info_dic(val): + geo_d = {} + if 'geo' in val.keys(): + geo_d['ctry'] = val['geo'].get('ctry', "unknown") + + bl_l = [] + for l in val.get('bl', []): + bl_l.append(l['n']) # TODO: shouldn't there be a check for v=1? + + tags_l = [] + for l in val.get('tags', []): + d = { + 'n': l, + 'c': val['tags'][l]['confidence'] + } + + tags_l.append(d) + + data = { + 'ip': val['_id'], + 'rep': val.get('rep', 0.0), + 'fmp': val.get('fmp', {'general': 0.0}), + 'hostname': (val.get('hostname', '') or '')[::-1], + 'ipblock': val.get('ipblock', ''), + 'bgppref': val.get('bgppref', ''), + 'asn': val.get('asn', []), + 'geo': geo_d, + 'bl': bl_l, + 'tags': tags_l + } + + return data + + +def get_basic_info_dic_short(val): + # only 'rep' and 'tags' fields + tags_l = [] + for l in val.get('tags', []): + d = { + 'n': l, + 'c': val['tags'][l]['confidence'] + } + tags_l.append(d) + + data = { + 'ip': val['_id'], + 'rep': val.get('rep', 0.0), + 'tags': tags_l + } + return data + +def get_basic_info_dic_v2(val): + geo_d = {} + if 'geo' in val.keys(): + geo_d['ctry'] = val['geo'].get('ctry', "unknown") + + bl_l = [] + for l in val.get('bl', []): + bl_l.append(l['n']) # TODO: shouldn't there be a check for v=1? + + tags_l = [] + for l in val.get('tags', []): + d = { + 'n' : l, + 'c' : val['tags'][l]['confidence'] + } + + tags_l.append(d) + + data = { + 'ip' : val['_id'], + 'rep' : val.get('rep', 0.0), + 'fmp' : val.get('fmp', {'general': 0.0}), + 'hostname' : (val.get('hostname', '') or '')[::-1], + 'ipblock' : val.get('ipblock', ''), + 'bgppref' : val.get('bgppref', ''), + 'asn' : val.get('asn',[]), + 'geo' : geo_d, + 'bl' : bl_l, + 'tags' : tags_l, + 'ts_last_update' : val.get('ts_last_update', ''), + 'ts_added' : val.get('ts_added', ''), + } + + return data + + +def find_ip_data(query, skip_n, limit): + return mongo.db.ip.find(query).skip(skip_n).limit(limit) + +def ip_to_warden_data(ipaddr): + ipint = ipstr2int(ipaddr) + ipinfo = mongo.db.ip.aggregate(pipeline = [ + { '$match': { '_id': ipint } }, + { '$unwind': '$events' }, + { '$group': { + '_id': { + 'date': '$events.date', + 'cat': '$events.cat' + }, + 'n_sum': { '$sum': '$events.n' }, + 'conns_sum': { '$sum': '$events.conns' }, + } + }, + { '$group': { + '_id': '$_id.date', + 'categories': { + "$push": { + "k": "$_id.cat", + "v": { + "n_sum": "$n_sum", + "conns_sum": "$conns_sum", + "nodes": "$nodes" + } + } + } + } + }, + { '$project': { + '_id': 0, + 'date': '$_id', + "categories": {"$arrayToObject": "$categories"} + } + }, + { '$sort': { 'date': 1 } } + ]) + + ipinfo_list = [doc for doc in ipinfo] + #ipinfo_list['_id'] = int2ipstr(ipinfo_list['_id']) + return ipinfo_list + + +def create_query_v2(data): + # Prepare 'find' part of the query + queries = [] + if data is None: + return None + + if "subnet" in data and data["subnet"] is not None and len(data["subnet"]) != 0: + subqueries = [] + for subnet in data["subnet"]: + subnet = ipaddress.IPv4Network(subnet, strict=False) + subnet_start = int(subnet.network_address) # IP addresses are stored as int + subnet_end = int(subnet.broadcast_address) + subqueries.append( {'$and': [{'_id': {'$gte': subnet_start}}, {'_id': {'$lte': subnet_end}}]} ) + queries.append({'$or': subqueries}) + + if "hostname" in data and data["hostname"] is not None and len(data["hostname"]) != 0: + subqueries = [] + for hostname in data["hostname"]: + hn = hostname[::-1] # Hostnames are stored reversed in DB to allow search by suffix as a range search + hn_end = hn[:-1] + chr(ord(hn[-1])+1) + subqueries.append( {'$and': [{'hostname': {'$gte': hn}}, {'hostname': {'$lt': hn_end}}]} ) + queries.append({'$or': subqueries}) + + if "country" in data and data["country"] is not None and len(data["country"]) != 0: + queries.append( { '$or': [{'geo.ctry': c.upper() } for c in data["country"]]} ) + + if "asn" in data and data["asn"] is not None and len(data["asn"]) != 0 : + subqueries = [] + for asn in data["asn"]: + # ASN is not stored in IP records - get list of BGP prefixes of the ASN and filter by these + asn = int(asn.lstrip("ASas")) + asrec = mongo.db.asn.find_one({'_id': asn}) + if asrec and 'bgppref' in asrec: + subqueries.append( {'bgppref': {'$in': asrec['bgppref']}} ) + else: + subqueries.append( {'_id': {'$exists': False}} ) # ASN not in DB, add query which is always false to get no results + op = '$and' if (data["asn_op"] == "AND") else '$or' + queries.append({op: subqueries}) + + if "source" in data and data["source"] is not None and len(data["source"]) != 0: + op = '$and' if (data["source_op"] == "AND") else '$or' + queries.append( {op: [{'_ttl.' + s.lower(): {'$exists': True}} for s in data["source"]]} ) + + if "cat" in data and data["cat"] is not None and len(data["cat"]) != 0: + op = '$and' if (data["cat_op"] == "AND") else '$or' + queries.append( {op: [{'events.cat': cat} for cat in data["cat"]]} ) + + if "node" in data and data["node"] is not None and len(data["node"]) != 0: + op = '$and' if (data["node_op"] == "AND") else '$or' + queries.append( {op: [{'events.node': node} for node in data["node"]]} ) + + if "blacklist" in data and data["blacklist"] is not None and len(data["blacklist"]) != 0: + op = '$and' if (data["bl_op"] == "AND") else '$or' + array = [{('dbl' if t == 'd' else 'bl'): {'$elemMatch': {'n': id, 'v': 1}}} for t,_,id in map(lambda s: s.partition(':'), data["blacklist"])] + queries.append( {op: array} ) + + if "tag" in data and data["tag"] is not None and len(data["tag"]) != 0: + op = '$and' if (data["tag_op"] == "AND") else '$or' + queries.append( {op: [{'tags.'+ tag_id: {'$exists': True}} for tag_id in data["tag"]]} ) + + if "whitelisted" in data and data["whitelisted"]: + queries.append( {'tags.whitelist': {'$exists': False}} ) + + query = {'$and': queries} if queries else None + return query \ No newline at end of file diff --git a/NERDweb/nerd_main.py b/NERDweb/nerd_main.py index c3f70661..28cfddc8 100644 --- a/NERDweb/nerd_main.py +++ b/NERDweb/nerd_main.py @@ -41,6 +41,8 @@ import userdb import ratelimit from userdb import get_user_info, authenticate_with_token, generate_unique_token +from com import attach_whois_data, app, mongo, mailer, validator_optional, strip_whitespace, clean_secret_data, conv_dates, \ + get_ip_info, get_basic_info_dic, get_basic_info_dic_short, find_ip_data, ip_to_warden_data # ***** Load configuration ***** @@ -161,37 +163,37 @@ # **** Create and initialize Flask application ***** -app = Flask(__name__) +# app = Flask(__name__) -app.secret_key = config.get('secret_key') +# app.secret_key = config.get('secret_key') -app.jinja_env.trim_blocks = True -app.jinja_env.lstrip_blocks = True +# app.jinja_env.trim_blocks = True +# app.jinja_env.lstrip_blocks = True -# Configuration of PyMongo -mongo_dbname = config.get('mongodb.dbname', 'nerd') -mongo_host = config.get('mongodb.host', 'localhost:27017') -if isinstance(mongo_host, list): - mongo_host = ','.join(mongo_host) -mongo_uri = "mongodb://{}/{}".format(mongo_host, mongo_dbname) -mongo_rs = config.get('mongodb.rs', None) -if mongo_rs: - mongo_uri += '?replicaSet='+mongo_rs -app.config['MONGO_URI'] = mongo_uri -print("MongoDB: Connecting to: {}".format(mongo_uri)) +# # Configuration of PyMongo +# mongo_dbname = config.get('mongodb.dbname', 'nerd') +# mongo_host = config.get('mongodb.host', 'localhost:27017') +# if isinstance(mongo_host, list): +# mongo_host = ','.join(mongo_host) +# mongo_uri = "mongodb://{}/{}".format(mongo_host, mongo_dbname) +# mongo_rs = config.get('mongodb.rs', None) +# if mongo_rs: +# mongo_uri += '?replicaSet='+mongo_rs +# app.config['MONGO_URI'] = mongo_uri +# print("MongoDB: Connecting to: {}".format(mongo_uri)) -mongo = PyMongo(app) +# mongo = PyMongo(app) -# Configuration of MAIL extension -app.config['MAIL_SERVER'] = config.get('mail.server', 'localhost') -app.config['MAIL_PORT'] = config.get('mail.port', '25') -app.config['MAIL_USE_TLS'] = config.get('mail.tls', False) -app.config['MAIL_USE_SSL'] = config.get('mail.ssl', False) -app.config['MAIL_USERNAME'] = config.get('mail.username', None) -app.config['MAIL_PASSWORD'] = config.get('mail.password', None) -app.config['MAIL_DEFAULT_SENDER'] = config.get('mail.sender', 'NERD ') +# # Configuration of MAIL extension +# app.config['MAIL_SERVER'] = config.get('mail.server', 'localhost') +# app.config['MAIL_PORT'] = config.get('mail.port', '25') +# app.config['MAIL_USE_TLS'] = config.get('mail.tls', False) +# app.config['MAIL_USE_SSL'] = config.get('mail.ssl', False) +# app.config['MAIL_USERNAME'] = config.get('mail.username', None) +# app.config['MAIL_PASSWORD'] = config.get('mail.password', None) +# app.config['MAIL_DEFAULT_SENDER'] = config.get('mail.sender', 'NERD ') -mailer = Mail(app) +# mailer = Mail(app) # Disable CSRF protection globally (it's OK to send search requests from anywhere) # FIXME: This disables it completeley, it would be better to rather disable it @@ -322,16 +324,16 @@ def misp_get_cluster_count(event): # The original one uses field.raw_data, but we need field.data # (raw_data only contain data loaded from Form (GET/POST), not those passed # to Form constructor using obj/data/kwargs parameters) -def validator_optional(form, field): - if not field.data: - field.errors[:] = [] - raise validators.StopValidation() +# def validator_optional(form, field): +# if not field.data: +# field.errors[:] = [] +# raise validators.StopValidation() # Filter to strip whitespaces in string -def strip_whitespace(s): - if isinstance(s, str): - s = s.strip() - return s +# def strip_whitespace(s): +# if isinstance(s, str): +# s = s.strip() +# return s # ***** Auxiliary functions ***** @@ -1681,183 +1683,108 @@ def api_user_info(): return Response(json.dumps(data), 200, mimetype='application/json') -def get_ip_info(ipaddr, full): - data = { - 'err_n': 400, - 'error': "No IP address specified", - 'ip': ipaddr - } - - if not ipaddr: - log_err.log('400_bad_request') - return False, Response(json.dumps(data), 400, mimetype='application/json') - - form = SingleIPForm(ip=ipaddr) - if not form.validate(): - log_err.log('400_bad_request') - data['error'] = "Bad IP address" - return False, Response(json.dumps(data), 400, mimetype='application/json') - - ipint = ipstr2int(form.ip.data) # Convert string IP to int - - if full: - ipinfo = mongo.db.ip.find_one({'_id': ipint}) - else: - ipinfo = mongo.db.ip.find_one({'_id': ipint}, - {'rep': 1, 'fmp': 1, 'hostname': 1, 'bgppref': 1, 'ipblock': 1, 'geo': 1, 'bl': 1, - 'tags': 1}) - if not ipinfo: - log_err.log('404_api_ip_not_found') - data['err_n'] = 404 - data['error'] = "IP address not found" - return False, Response(json.dumps(data), 404, mimetype='application/json') - - ipinfo['_id'] = int2ipstr(ipinfo['_id']) # Convert int IP to string - - attach_whois_data(ipinfo, full) - return True, ipinfo - - -def conv_dates(rec): - """Convert datetimes in a record to YYYY-MM-DDTMM:HH:SS string""" - for key in ('ts_added', 'ts_last_update'): - if key in rec and isinstance(rec[key], datetime): - rec[key] = rec[key].strftime("%Y-%m-%dT%H:%M:%S") - - -def attach_whois_data(ipinfo, full): - """ - Attach records of related entities to given IP record. - - If full==True, attach full records of BGP prefix, ASNs, IP block, Org entities (as 'bgppref, 'asn', 'ipblock' and 'org' keys), - otherwise only attach list of ASN numbers (as 'asn' key). - """ - if not full: - # Only attach ASN number(s) - if 'bgppref' in ipinfo: - bgppref_rec = mongo.db.bgppref.find_one({'_id': ipinfo['bgppref']}, {'asn': 1}) - if bgppref_rec is None: - print("ERROR: Can't find BGP prefix '{}' in database (trying to enrich IP {})".format(ipinfo['bgppref'], ipinfo['_id'])) - return - if 'asn' in bgppref_rec: - ipinfo['asn'] = bgppref_rec['asn'] - return - - # Full - attach full records of related BGP prefix, ASNs, IP block, Org - # IP->BGPpref - if 'bgppref' in ipinfo: - bgppref_rec = clean_secret_data(mongo.db.bgppref.find_one({'_id': ipinfo['bgppref']})) - if bgppref_rec is None: - print("ERROR: Can't find BGP prefix '{}' in database (trying to enrich IP {})".format(ipinfo['bgppref'], ipinfo['_id'])) - else: - # BGPpref->ASN(s) - asn_list = [] - for asn in bgppref_rec['asn']: - asn_rec = clean_secret_data(mongo.db.asn.find_one({'_id': asn})) - if asn_rec is None: - print("ERROR: Can't find ASN '{}' in database (trying to enrich IP {}, bgppref {})".format(asn, ipinfo['_id'], bgppref_rec['_id'])) - else: - # ASN->Org - if 'org' in asn_rec: - org_rec = clean_secret_data(mongo.db.org.find_one({'_id': asn_rec['org']})) - if org_rec is None: - print( - "ERROR: Can't find Org '{}' in database (trying to enrich IP {}, bgppref {}, ASN {})".format( - asn_rec['org'], ipinfo['_id'], bgppref_rec['_id'], asn)) - else: - conv_dates(org_rec) - asn_rec['org'] = org_rec - - del asn_rec['bgppref'] - conv_dates(asn_rec) - asn_list.append(asn_rec) - - del bgppref_rec['asn'] - conv_dates(bgppref_rec) - ipinfo['bgppref'] = bgppref_rec - ipinfo['asn'] = asn_list - - # IP->ipblock - if 'ipblock' in ipinfo: - ipblock_rec = clean_secret_data(mongo.db.ipblock.find_one({'_id': ipinfo['ipblock']})) - if ipblock_rec is None: - print("ERROR: Can't find IP block '{}' in database (trying to enrich IP {})".format(ipinfo['ipblock'], - ipinfo['_id'])) - else: - # ipblock->org - if "org" in ipblock_rec: - org_rec = clean_secret_data(mongo.db.org.find_one({'_id': ipblock_rec['org']})) - if org_rec is None: - print("ERROR: Can't find Org '{}' in database (trying to enrich IP {}, ipblock '{}')".format( - ipblock_rec['org'], ipinfo['_id'], ipblock_rec['_id'])) - else: - conv_dates(org_rec) - ipblock_rec['org'] = org_rec - - conv_dates(ipblock_rec) - ipinfo['ipblock'] = ipblock_rec - - -def clean_secret_data(data): - """Remove all keys starting with '_' (except '_id') from dict.""" - if data is not None: - for i in list(data): - if i.startswith("_") and i != "_id": - del data[i] - return data +# def get_ip_info(ipaddr, full): +# data = { +# 'err_n': 400, +# 'error': "No IP address specified", +# 'ip': ipaddr +# } + +# if not ipaddr: +# log_err.log('400_bad_request') +# return False, Response(json.dumps(data), 400, mimetype='application/json') + +# form = SingleIPForm(ip=ipaddr) +# if not form.validate(): +# log_err.log('400_bad_request') +# data['error'] = "Bad IP address" +# return False, Response(json.dumps(data), 400, mimetype='application/json') + +# ipint = ipstr2int(form.ip.data) # Convert string IP to int + +# if full: +# ipinfo = mongo.db.ip.find_one({'_id': ipint}) +# else: +# ipinfo = mongo.db.ip.find_one({'_id': ipint}, +# {'rep': 1, 'fmp': 1, 'hostname': 1, 'bgppref': 1, 'ipblock': 1, 'geo': 1, 'bl': 1, +# 'tags': 1}) +# if not ipinfo: +# log_err.log('404_api_ip_not_found') +# data['err_n'] = 404 +# data['error'] = "IP address not found" +# return False, Response(json.dumps(data), 404, mimetype='application/json') + +# ipinfo['_id'] = int2ipstr(ipinfo['_id']) # Convert int IP to string + +# attach_whois_data(ipinfo, full) +# return True, ipinfo + +# def conv_dates(rec): +# """Convert datetimes in a record to YYYY-MM-DDTMM:HH:SS string""" +# for key in ('ts_added', 'ts_last_update'): +# if key in rec and isinstance(rec[key], datetime): +# rec[key] = rec[key].strftime("%Y-%m-%dT%H:%M:%S") + +# def clean_secret_data(data): +# """Remove all keys starting with '_' (except '_id') from dict.""" +# if data is not None: +# for i in list(data): +# if i.startswith("_") and i != "_id": +# del data[i] +# return data # ***** NERD API BasicInfo - helper funcs ***** -def get_basic_info_dic(val): - geo_d = {} - if 'geo' in val.keys(): - geo_d['ctry'] = val['geo'].get('ctry', "unknown") - - bl_l = [] - for l in val.get('bl', []): - bl_l.append(l['n']) # TODO: shouldn't there be a check for v=1? - - tags_l = [] - for l in val.get('tags', []): - d = { - 'n': l, - 'c': val['tags'][l]['confidence'] - } - - tags_l.append(d) - - data = { - 'ip': val['_id'], - 'rep': val.get('rep', 0.0), - 'fmp': val.get('fmp', {'general': 0.0}), - 'hostname': (val.get('hostname', '') or '')[::-1], - 'ipblock': val.get('ipblock', ''), - 'bgppref': val.get('bgppref', ''), - 'asn': val.get('asn', []), - 'geo': geo_d, - 'bl': bl_l, - 'tags': tags_l - } - - return data - - -def get_basic_info_dic_short(val): - # only 'rep' and 'tags' fields - tags_l = [] - for l in val.get('tags', []): - d = { - 'n': l, - 'c': val['tags'][l]['confidence'] - } - tags_l.append(d) - - data = { - 'ip': val['_id'], - 'rep': val.get('rep', 0.0), - 'tags': tags_l - } - return data +# def get_basic_info_dic(val): +# geo_d = {} +# if 'geo' in val.keys(): +# geo_d['ctry'] = val['geo'].get('ctry', "unknown") + +# bl_l = [] +# for l in val.get('bl', []): +# bl_l.append(l['n']) # TODO: shouldn't there be a check for v=1? + +# tags_l = [] +# for l in val.get('tags', []): +# d = { +# 'n': l, +# 'c': val['tags'][l]['confidence'] +# } + +# tags_l.append(d) + +# data = { +# 'ip': val['_id'], +# 'rep': val.get('rep', 0.0), +# 'fmp': val.get('fmp', {'general': 0.0}), +# 'hostname': (val.get('hostname', '') or '')[::-1], +# 'ipblock': val.get('ipblock', ''), +# 'bgppref': val.get('bgppref', ''), +# 'asn': val.get('asn', []), +# 'geo': geo_d, +# 'bl': bl_l, +# 'tags': tags_l +# } + +# return data + + +# def get_basic_info_dic_short(val): +# # only 'rep' and 'tags' fields +# tags_l = [] +# for l in val.get('tags', []): +# d = { +# 'n': l, +# 'c': val['tags'][l]['confidence'] +# } +# tags_l.append(d) + +# data = { +# 'ip': val['_id'], +# 'rep': val.get('rep', 0.0), +# 'tags': tags_l +# } +# return data # ***** NERD API BasicInfo ***** @@ -2322,150 +2249,150 @@ def get_shodan_response(ipaddr=None): # ********** # ***************************** API v2 helper functions ******************************************** -def create_query_v2(data): - # Prepare 'find' part of the query - queries = [] - if data is None: - return None +# def create_query_v2(data): +# # Prepare 'find' part of the query +# queries = [] +# if data is None: +# return None - if "subnet" in data and data["subnet"] is not None and len(data["subnet"]) != 0: - subqueries = [] - for subnet in data["subnet"]: - subnet = ipaddress.IPv4Network(subnet, strict=False) - subnet_start = int(subnet.network_address) # IP addresses are stored as int - subnet_end = int(subnet.broadcast_address) - subqueries.append( {'$and': [{'_id': {'$gte': subnet_start}}, {'_id': {'$lte': subnet_end}}]} ) - queries.append({'$or': subqueries}) - - if "hostname" in data and data["hostname"] is not None and len(data["hostname"]) != 0: - subqueries = [] - for hostname in data["hostname"]: - hn = hostname[::-1] # Hostnames are stored reversed in DB to allow search by suffix as a range search - hn_end = hn[:-1] + chr(ord(hn[-1])+1) - subqueries.append( {'$and': [{'hostname': {'$gte': hn}}, {'hostname': {'$lt': hn_end}}]} ) - queries.append({'$or': subqueries}) - - if "country" in data and data["country"] is not None and len(data["country"]) != 0: - queries.append( { '$or': [{'geo.ctry': c.upper() } for c in data["country"]]} ) - - if "asn" in data and data["asn"] is not None and len(data["asn"]) != 0 : - subqueries = [] - for asn in data["asn"]: - # ASN is not stored in IP records - get list of BGP prefixes of the ASN and filter by these - asn = int(asn.lstrip("ASas")) - asrec = mongo.db.asn.find_one({'_id': asn}) - if asrec and 'bgppref' in asrec: - subqueries.append( {'bgppref': {'$in': asrec['bgppref']}} ) - else: - subqueries.append( {'_id': {'$exists': False}} ) # ASN not in DB, add query which is always false to get no results - op = '$and' if (data["asn_op"] == "AND") else '$or' - queries.append({op: subqueries}) - - if "source" in data and data["source"] is not None and len(data["source"]) != 0: - op = '$and' if (data["source_op"] == "AND") else '$or' - queries.append( {op: [{'_ttl.' + s.lower(): {'$exists': True}} for s in data["source"]]} ) - - if "cat" in data and data["cat"] is not None and len(data["cat"]) != 0: - op = '$and' if (data["cat_op"] == "AND") else '$or' - queries.append( {op: [{'events.cat': cat} for cat in data["cat"]]} ) - - if "node" in data and data["node"] is not None and len(data["node"]) != 0: - op = '$and' if (data["node_op"] == "AND") else '$or' - queries.append( {op: [{'events.node': node} for node in data["node"]]} ) - - if "blacklist" in data and data["blacklist"] is not None and len(data["blacklist"]) != 0: - op = '$and' if (data["bl_op"] == "AND") else '$or' - array = [{('dbl' if t == 'd' else 'bl'): {'$elemMatch': {'n': id, 'v': 1}}} for t,_,id in map(lambda s: s.partition(':'), data["blacklist"])] - queries.append( {op: array} ) - - if "tag" in data and data["tag"] is not None and len(data["tag"]) != 0: - op = '$and' if (data["tag_op"] == "AND") else '$or' - queries.append( {op: [{'tags.'+ tag_id: {'$exists': True}} for tag_id in data["tag"]]} ) - - if "whitelisted" in data and data["whitelisted"]: - queries.append( {'tags.whitelist': {'$exists': False}} ) - - query = {'$and': queries} if queries else None - return query - -def get_basic_info_dic_v2(val): - geo_d = {} - if 'geo' in val.keys(): - geo_d['ctry'] = val['geo'].get('ctry', "unknown") - - bl_l = [] - for l in val.get('bl', []): - bl_l.append(l['n']) # TODO: shouldn't there be a check for v=1? - - tags_l = [] - for l in val.get('tags', []): - d = { - 'n' : l, - 'c' : val['tags'][l]['confidence'] - } - - tags_l.append(d) - - data = { - 'ip' : val['_id'], - 'rep' : val.get('rep', 0.0), - 'fmp' : val.get('fmp', {'general': 0.0}), - 'hostname' : (val.get('hostname', '') or '')[::-1], - 'ipblock' : val.get('ipblock', ''), - 'bgppref' : val.get('bgppref', ''), - 'asn' : val.get('asn',[]), - 'geo' : geo_d, - 'bl' : bl_l, - 'tags' : tags_l, - 'ts_last_update' : val.get('ts_last_update', ''), - 'ts_added' : val.get('ts_added', ''), - } - - return data - -def find_ip_data(query, skip_n, limit): - return mongo.db.ip.find(query).skip(skip_n).limit(limit) - -def ip_to_warden_data(ipaddr): - ipint = ipstr2int(ipaddr) - ipinfo = mongo.db.ip.aggregate(pipeline = [ - { '$match': { '_id': ipint } }, - { '$unwind': '$events' }, - { '$group': { - '_id': { - 'date': '$events.date', - 'cat': '$events.cat' - }, - 'n_sum': { '$sum': '$events.n' }, - 'conns_sum': { '$sum': '$events.conns' }, - } - }, - { '$group': { - '_id': '$_id.date', - 'categories': { - "$push": { - "k": "$_id.cat", - "v": { - "n_sum": "$n_sum", - "conns_sum": "$conns_sum", - "nodes": "$nodes" - } - } - } - } - }, - { '$project': { - '_id': 0, - 'date': '$_id', - "categories": {"$arrayToObject": "$categories"} - } - }, - { '$sort': { 'date': 1 } } - ]) - - ipinfo_list = [doc for doc in ipinfo] - #ipinfo_list['_id'] = int2ipstr(ipinfo_list['_id']) - return ipinfo_list +# if "subnet" in data and data["subnet"] is not None and len(data["subnet"]) != 0: +# subqueries = [] +# for subnet in data["subnet"]: +# subnet = ipaddress.IPv4Network(subnet, strict=False) +# subnet_start = int(subnet.network_address) # IP addresses are stored as int +# subnet_end = int(subnet.broadcast_address) +# subqueries.append( {'$and': [{'_id': {'$gte': subnet_start}}, {'_id': {'$lte': subnet_end}}]} ) +# queries.append({'$or': subqueries}) + +# if "hostname" in data and data["hostname"] is not None and len(data["hostname"]) != 0: +# subqueries = [] +# for hostname in data["hostname"]: +# hn = hostname[::-1] # Hostnames are stored reversed in DB to allow search by suffix as a range search +# hn_end = hn[:-1] + chr(ord(hn[-1])+1) +# subqueries.append( {'$and': [{'hostname': {'$gte': hn}}, {'hostname': {'$lt': hn_end}}]} ) +# queries.append({'$or': subqueries}) + +# if "country" in data and data["country"] is not None and len(data["country"]) != 0: +# queries.append( { '$or': [{'geo.ctry': c.upper() } for c in data["country"]]} ) + +# if "asn" in data and data["asn"] is not None and len(data["asn"]) != 0 : +# subqueries = [] +# for asn in data["asn"]: +# # ASN is not stored in IP records - get list of BGP prefixes of the ASN and filter by these +# asn = int(asn.lstrip("ASas")) +# asrec = mongo.db.asn.find_one({'_id': asn}) +# if asrec and 'bgppref' in asrec: +# subqueries.append( {'bgppref': {'$in': asrec['bgppref']}} ) +# else: +# subqueries.append( {'_id': {'$exists': False}} ) # ASN not in DB, add query which is always false to get no results +# op = '$and' if (data["asn_op"] == "AND") else '$or' +# queries.append({op: subqueries}) + +# if "source" in data and data["source"] is not None and len(data["source"]) != 0: +# op = '$and' if (data["source_op"] == "AND") else '$or' +# queries.append( {op: [{'_ttl.' + s.lower(): {'$exists': True}} for s in data["source"]]} ) + +# if "cat" in data and data["cat"] is not None and len(data["cat"]) != 0: +# op = '$and' if (data["cat_op"] == "AND") else '$or' +# queries.append( {op: [{'events.cat': cat} for cat in data["cat"]]} ) + +# if "node" in data and data["node"] is not None and len(data["node"]) != 0: +# op = '$and' if (data["node_op"] == "AND") else '$or' +# queries.append( {op: [{'events.node': node} for node in data["node"]]} ) + +# if "blacklist" in data and data["blacklist"] is not None and len(data["blacklist"]) != 0: +# op = '$and' if (data["bl_op"] == "AND") else '$or' +# array = [{('dbl' if t == 'd' else 'bl'): {'$elemMatch': {'n': id, 'v': 1}}} for t,_,id in map(lambda s: s.partition(':'), data["blacklist"])] +# queries.append( {op: array} ) + +# if "tag" in data and data["tag"] is not None and len(data["tag"]) != 0: +# op = '$and' if (data["tag_op"] == "AND") else '$or' +# queries.append( {op: [{'tags.'+ tag_id: {'$exists': True}} for tag_id in data["tag"]]} ) + +# if "whitelisted" in data and data["whitelisted"]: +# queries.append( {'tags.whitelist': {'$exists': False}} ) + +# query = {'$and': queries} if queries else None +# return query + +# def get_basic_info_dic_v2(val): +# geo_d = {} +# if 'geo' in val.keys(): +# geo_d['ctry'] = val['geo'].get('ctry', "unknown") + +# bl_l = [] +# for l in val.get('bl', []): +# bl_l.append(l['n']) # TODO: shouldn't there be a check for v=1? + +# tags_l = [] +# for l in val.get('tags', []): +# d = { +# 'n' : l, +# 'c' : val['tags'][l]['confidence'] +# } + +# tags_l.append(d) + +# data = { +# 'ip' : val['_id'], +# 'rep' : val.get('rep', 0.0), +# 'fmp' : val.get('fmp', {'general': 0.0}), +# 'hostname' : (val.get('hostname', '') or '')[::-1], +# 'ipblock' : val.get('ipblock', ''), +# 'bgppref' : val.get('bgppref', ''), +# 'asn' : val.get('asn',[]), +# 'geo' : geo_d, +# 'bl' : bl_l, +# 'tags' : tags_l, +# 'ts_last_update' : val.get('ts_last_update', ''), +# 'ts_added' : val.get('ts_added', ''), +# } + +# return data + +# def find_ip_data(query, skip_n, limit): +# return mongo.db.ip.find(query).skip(skip_n).limit(limit) + +# def ip_to_warden_data(ipaddr): +# ipint = ipstr2int(ipaddr) +# ipinfo = mongo.db.ip.aggregate(pipeline = [ +# { '$match': { '_id': ipint } }, +# { '$unwind': '$events' }, +# { '$group': { +# '_id': { +# 'date': '$events.date', +# 'cat': '$events.cat' +# }, +# 'n_sum': { '$sum': '$events.n' }, +# 'conns_sum': { '$sum': '$events.conns' }, +# } +# }, +# { '$group': { +# '_id': '$_id.date', +# 'categories': { +# "$push": { +# "k": "$_id.cat", +# "v": { +# "n_sum": "$n_sum", +# "conns_sum": "$conns_sum", +# "nodes": "$nodes" +# } +# } +# } +# } +# }, +# { '$project': { +# '_id': 0, +# 'date': '$_id', +# "categories": {"$arrayToObject": "$categories"} +# } +# }, +# { '$sort': { 'date': 1 } } +# ]) + +# ipinfo_list = [doc for doc in ipinfo] +# #ipinfo_list['_id'] = int2ipstr(ipinfo_list['_id']) +# return ipinfo_list # additional data for selected users that belong to group diff --git a/NERDweb/user_management.py b/NERDweb/user_management.py index 4db27511..c14f5843 100644 --- a/NERDweb/user_management.py +++ b/NERDweb/user_management.py @@ -9,10 +9,13 @@ from itsdangerous import URLSafeTimedSerializer, BadSignature, SignatureExpired from flask_dance.consumer import OAuth2ConsumerBlueprint import jwt +import os +import sys +import common.config # needed overridden render_template method because it passes some needed attributes to Jinja templates -from nerd_main import render_template, BASE_URL, config, mailer +from com import mailer from userdb import create_user, get_user_data_for_login, get_user_by_email, verify_user, get_verification_email_sent, \ set_verification_email_sent, set_last_login, get_user_name, set_new_password, check_if_user_exists, get_user_by_id @@ -23,6 +26,22 @@ user_management = Blueprint("user_management", __name__, static_folder="static", template_folder="templates") +DEFAULT_CONFIG_FILE = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "/etc/nerd/nerdweb.yml")) + +if len(sys.argv) >= 2: + cfg_file = sys.argv[1] +else: + cfg_file = DEFAULT_CONFIG_FILE +cfg_dir = os.path.dirname(os.path.abspath(cfg_file)) + +# Read web-specific config (nerdweb.cfg) +config = common.config.read_config(cfg_file) +# Read common config (nerd.cfg) and combine them together +common_cfg_file = os.path.join(cfg_dir, config.get('common_config')) +config.update(common.config.read_config(common_cfg_file)) + +config.testing = True + # ***** Util functions ***** def generate_token(user_email): """ Generates random token for email verification or password reset. """