diff --git a/NERDweb/api_v2.py b/NERDweb/api_v2.py new file mode 100644 index 00000000..8e343d58 --- /dev/null +++ b/NERDweb/api_v2.py @@ -0,0 +1,1082 @@ +from flask import Blueprint, flash, redirect, session, g, make_response, \ + current_app, url_for, request, make_response, Response, jsonify, url_for +import requests +from flask_pymongo import pymongo, PyMongo +from datetime import datetime, timedelta, timezone +import jwt +from jwt import PyJWKClient +from jwt.exceptions import DecodeError +from werkzeug.exceptions import InternalServerError, Unauthorized +import os +import sys +import common.config +from auth import token_required +from common.utils import ipstr2int, int2ipstr, parse_rfc_time +import json +from bcrypt import hashpw, checkpw, gensalt +import subprocess +import re +from flasgger import swag_from +from flask_mail import Mail, Message +from requests_oauthlib import OAuth1Session, OAuth2Session + +from user_management import get_hashed_password, generate_token, verify_email_token, \ + generate_jwt_token, confirm_jwt_token + +from com import attach_whois_data, get_ip_info, mailer, mongo, get_basic_info_dic, get_basic_info_dic_short, get_basic_info_dic_v2, find_ip_data, ip_to_warden_data, create_query_v2 +from userdb import get_user_info, authenticate_with_token, generate_unique_token, \ + get_user_by_id, create_user, verify_user, set_last_login, get_user_name, \ + set_new_password, get_users_admin, set_verification_email_sent, set_new_roles, \ + delete_user, just_verify_user_by_id, set_api_v1_token, get_user_by_email + +DEFAULT_CONFIG_FILE = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "/etc/nerd/nerdweb.yml")) + +if len(sys.argv) >= 2: + cfg_file = sys.argv[1] +else: + cfg_file = DEFAULT_CONFIG_FILE +cfg_dir = os.path.dirname(os.path.abspath(cfg_file)) + +# Read web-specific config (nerdweb.cfg) +config = common.config.read_config(cfg_file) +# Read common config (nerd.cfg) and combine them together +common_cfg_file = os.path.join(cfg_dir, config.get('common_config')) +config.update(common.config.read_config(common_cfg_file)) + +config.testing = True + +# url for redirects +APP_BASE_URL = config.get('app_base_url') + +api_v2 = Blueprint("api_v2", __name__, static_folder="static", template_folder="templates") + +def check_email(email): + return re.fullmatch(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,7}\b', email) + +def send_password_reset_email(user_email): + if not config.get('login.request-email', None): + return dict(message='Error while sendig email.'), 402 + token = generate_jwt_token(user_email) + password_reset_url = url_for('api_v2.password_reset', token=token, _external=True) + msg = Message(subject="[NERD] Password reset request", + recipients=[user_email], + reply_to=current_app.config.get('MAIL_DEFAULT_SENDER'), + body=f"Dear user,\n\nyou can reset your password by clicking on this link:" + f"\n\n{password_reset_url}\n\nThank you,\nNERD administrator", + ) + mailer.send(msg) + +######################################################## +# NERD ENDPOINTS # +######################################################## + +# ***** NERD API BasicInfo ***** +@api_v2.route('/ip/', methods=['GET']) +def get_basic_info_v2(ipaddr=None): + """Basic info for a single IP address + --- + parameters: + - name: ipaddr + in: path + type: string + required: true + description: IP address in IPv4 format + responses: + 200: + description: An object containing basic info about IP + """ + ret, val = get_ip_info(ipaddr, False) + if not ret: + return val # val is an error Response + + binfo = get_basic_info_dic(val) + + return Response(json.dumps(binfo), 200, mimetype='application/json') + +# ***** NERD API IPSearch ***** +@api_v2.route('/search/ip', methods=['POST']) +@swag_from('./api_v2_swag/search.yml', validation=True) +def ip_search_v2(full = False): + err = {} + + # Get output format + output = request.json + query = create_query_v2(output) + + if "page" not in output: + output["page"] = 1 + + if "limit" not in output: + output["limit"] = 20 + + if "order" not in output: + output["order"] = "desc" + + if "sort" not in output or output["sort"] is None: + output["sort"] = "rep" + elif output["sort"] == "ip": + output["sort"] = "_id" + + try: + results = find_ip_data(query, (output["page"] - 1) * output["limit"], output["limit"]) # note: limit=0 means no limit + results.sort(output["sort"], 1 if output["order"] == "asc" else -1) + results = list(results) + except pymongo.errors.ServerSelectionTimeoutError: + log_err.log('503_db_error') + err['err_n'] = 503 + err['error'] = 'Database connection error' + resp = Response(json.dumps(err), 503, mimetype='application/json') + return resp + + # Return results + if output == "list": + resp = Response(''.join(int2ipstr(res['_id'])+'\n' for res in results), 200, mimetype='text/plain') + return resp + + # Convert _id from int to dotted-decimal string + for res in results: + res['_id'] = int2ipstr(res['_id']) + + lres = [] + if output == "short": + for res in results: + lres.append(get_basic_info_dic_short(res)) + else: + for res in results: + attach_whois_data(res, full) + lres.append(get_basic_info_dic_v2(res)) + + resp = Response(json.dumps(lres, default=str), 200, mimetype='application/json') + return resp + +# ***** NERD API Details ***** +@api_v2.route('/details/', methods=['GET']) +def get_full_info(ipaddr=None): + """Detailed info for a single IP address + --- + parameters: + - name: ipaddr + in: path + type: string + required: true + description: IP address in IPv4 format + responses: + 200: + description: An object containing detailed info about IP + """ + ret, val = get_ip_info(ipaddr, True) + if not ret: + return val # val is an error Response + + data = { + 'ip' : val['_id'], + 'rep' : val.get('rep', 0.0), + 'fmp' : val.get('fmp', {'general': 0.0}), + 'hostname' : (val.get('hostname', '') or '')[::-1], + 'ipblock' : val.get('ipblock', ''), + 'bgppref' : val.get('bgppref', ''), + 'asn' : val.get('asn',[]), + 'geo' : val.get('geo', None), + 'ts_added' : val['ts_added'].strftime("%Y-%m-%dT%H:%M:%S"), + 'ts_last_update' : val['ts_last_update'].strftime("%Y-%m-%dT%H:%M:%S"), + 'last_activity' : val['last_activity'].strftime("%Y-%m-%dT%H:%M:%S") if 'last_activity' in val else None, + 'bl' : [ { + 'name': bl['n'], + 'last_check': bl['t'].strftime("%Y-%m-%dT%H:%M:%S"), + 'last_result': True if bl['v'] else False, + 'history': [t.strftime("%Y-%m-%d") for t in bl['h']] + } for bl in val.get('bl', []) ], + 'events' : ip_to_warden_data(ipaddr), + 'misp_events' : val.get('misp_events', []), + 'events_meta' : { + 'total': val.get('events_meta', {}).get('total', 0.0), + 'total1': val.get('events_meta', {}).get('total1', 0.0), + 'total7': val.get('events_meta', {}).get('total7', 0.0), + 'total30': val.get('events_meta', {}).get('total30', 0.0), + }, + 'dshield' : val.get('dshield', []), + 'otx_pulses' : val.get('otx_pulses', []), + 'tags': val.get('tags', []), + + } + + return Response(json.dumps(data, default=str), 200, mimetype='application/json') + + + +######################################################## +# USER ENDPOINTS # +######################################################## +@api_v2.route('/login/devel') +def login_devel_v2(): + if not config.testing: + return flask.abort(404) + out = {} + out['user'] = { + 'login_type': 'devel', + 'id': 'devel_admin', + } + out['exp'] = datetime.utcnow()+timedelta(hours=4) + encoded_jwt = jwt.encode(out, config.get('secret_key'), algorithm="HS256") + return encoded_jwt + +# User login +@api_v2.route('/login', methods=['POST']) +@swag_from('./api_v2_swag/login.yml', validation=True) +def login_user_v2(): + try: + data = request.json + if not data: + return { + "message": "Please provide user details", + "data": None, + "error": "Bad request" + }, 400 + user = get_user_by_id("local:" + data["email"]) + + if user is None: + user = get_user_by_email(data["email"]) + if user is None: + return dict(message='User does not exist.'), 404 + return dict(message='Local user does not exist but provided email is associated with provider: ' + user['id'].split(":")[0]), 404 + + if not checkpw(data["password"].encode('utf-8'), user['password'].encode('utf-8')): + return dict(message='Bad password.'), 400 + try: + out = {} + out['user'] = { + 'login_type': 'local', + 'id': user['id'], + 'email': user['email'], + } + out['exp'] = datetime.utcnow()+timedelta(hours=4) + token = jwt.encode(out, config.get('secret_key'), algorithm="HS256") + out2 = {} + out2['user'] = { + 'login_type': 'local', + 'id': user['id'], + 'email': user['email'], + } + out2['exp'] = datetime.utcnow()+timedelta(hours=72) + refreshToken = jwt.encode(out2, config.get('secret_key'), algorithm="HS256") + set_last_login(datetime.utcnow(), user['id']) + return Response(json.dumps([token, refreshToken], default=str), 200, mimetype='application/json') + + except Exception as e: + return { + "error": "Something went wrong", + "message": str(e) + }, 500 + return { + "message": "Error fetching auth token!, invalid email or password", + "data": None, + "error": "Unauthorized" + }, 404 + except Exception as e: + return { + "message": "Something went wrong!", + "error": str(e), + "data": None + }, 500 + +@api_v2.route('/register', methods=['POST']) +@swag_from('./api_v2_swag/register.yml', validation=True) +def register_user_v2(): + data = request.json + if not check_email(data["email"]): + return dict(message='Wrong email format.'), 400 + if len(data["password"]) < 8: + return dict(message='Password not long enough.'), 400 + + hashed_password = get_hashed_password(data["password"]) + res = create_user(data["email"], hashed_password, "local", data["name"], data["organization"]) + if isinstance(res, Exception): + if res.args[0].startswith("duplicate key") and "Key (id)" in res.args[0]: #TODO? neexistuje lep�� zp�sob kontroly typu cyhby? Pr� na to speci�ln� typ Exc asi nen� + return dict(message=f"User with email address {data['email']} already exists! You can either log in or try to " + f"reset your password in log-in section."), 400 + else: + return dict(message=f"ERROR in register_user(): Something has failed during registration process: {res.args}"), 400 + else: + out = {} + out['user'] = { + 'email': data['email'], + } + out['exp'] = datetime.utcnow()+timedelta(hours=24) + encoded_jwt = jwt.encode(out, config.get('secret_key'), algorithm="HS256") + msg = Message(subject="[NERD] Account created", + recipients=[data["email"]], + reply_to=current_app.config.get('MAIL_DEFAULT_SENDER'), + ) + msg.body = f"NERD - Network Entity Reputation Database \nEmail address verification \n>Dear user {data['email']}, \nyou can activate your NERD account by clicking the link below (the link is valid for 24 hours): \n{config.get('email_web')}/verify?accessToken={encoded_jwt} \nNERD administrator" + msg.html = f"

NERD

Network Entity Reputation Database

Email address verification

Dear user {data['email']},

you can activate your NERD account by clicking this LINK (the link is valid for 24 hours).


NERD administrator

" + mailer.send(msg) + set_verification_email_sent(datetime.utcnow(), "local:" + data["email"]) + return dict(message=f"Account created for {data['email']}!"), 200 + +@api_v2.route('/verify', methods=['POST']) +def verify_email_address(): + """Email verification after registration + --- + consumes: + - application/json + parameters: + - in: body + name: body + description: Request body for endpoint + required: true + schema: + type: object + properties: + accessToken: + type: string + description: Email token provided after user registration + example: "ey...." + responses: + 200: + description: Successful email verification + 400: + description: accessToken missing + 500: + description: Internal server error + """ + data = request.json + if not data["accessToken"]: + return dict(message='No access token.'), 400 + try: + out = jwt.decode(data["accessToken"], config.get('secret_key'), algorithms=["HS256"]) + except Exception as e: + return { + "message": "Something went wrong", + "data": None, + "error": str(e) + }, 500 + try: + verify_user("local:" + out["user"]["email"]) + except Exception as e: + return { + "message": "Something went wrong", + "data": None, + "error": str(e) + }, 500 + return dict(message=f"Email verified for {out['user']['email']}!"), 200 + + +# gets user info extracted from JWT +@api_v2.route('/me', methods=['GET']) +@token_required +def me_info(current_user): + """User profile info + --- + security: + - OAuth2: [user] + responses: + 200: + description: User info fetched + """ + return jsonify({ "email": current_user["email"], "groups": current_user["groups"], "name": current_user["name"], "org": current_user["org"], "type": current_user["id"].split(":")[0], "api_v1_token": current_user["api_token"]}), 200 + + +@api_v2.route('/reset_password', methods=['POST']) +def password_reset_request(): + """Local user password reset request + --- + consumes: + - application/json + parameters: + - in: body + name: body + description: Request body for endpoint + required: true + schema: + type: object + properties: + email: + type: string + description: Local user email + example: "user@email.com" + responses: + 200: + description: Password reset email sent + """ + data = request.json + user = get_user_by_id("local:" + data["email"]) + if user is None: + return dict(message='User with this email does not exist.'), 401 + else: + send_password_reset_email(data["email"]) + return dict(message='Email with password reset link was sent!'), 200 + +@api_v2.route("/password_reset_request/", methods=['GET']) +def password_reset(token): + """Password reser redirect URI form email + --- + parameters: + - name: token + in: path + type: string + required: true + description: Token form email + responses: + 401: + description: User not found + """ + user = confirm_jwt_token(token) + if user is None or user['email'] is None: + return dict(message='User not found'), 401 + + return redirect(APP_BASE_URL + "/nerd2/password-reset?token=" + token) + +@api_v2.route("/password_reset_from_token", methods=['POST']) +def password_reset_token(): + """Local user password reset action + --- + consumes: + - application/json + parameters: + - in: body + name: body + description: Request body for endpoint + required: true + schema: + type: object + properties: + token: + type: string + description: Token from email + example: "ey..." + responses: + 200: + description: Password has been successfully changed + 400: + description: Problem when reseting password + 404: + description: User not found + """ + data = request.json + token = data["token"] + user = confirm_jwt_token(token) + if user is None or user['email'] is None: + return dict(message='User not found'), 404 + + if len(data["password"]) < 8: + return dict(message='Password not long enough'), 400 + + result = set_new_password(get_hashed_password(data["password"]), "local:" + user['email']) + if result is not None: + # exception occurred + return dict(message="Password reset failed. Please, try again and if the problem persists, contact NERD administrator."), 400 + else: + return dict(message="Password has been successfully changed!"), 200 + + +@api_v2.route("/change_password", methods=['POST']) +@token_required +def password_change(current_user): + """Local user password change + --- + security: + - OAuth2: [user] + consumes: + - application/json + parameters: + - in: body + name: body + description: Request body for endpoint + required: true + schema: + type: object + properties: + passOld: + type: string + description: old passowrd + example: "password123" + password: + type: string + description: new password + example: "password12" + responses: + 200: + description: Password has been successfully changed + 400: + description: Problem when reseting password + 404: + description: Wrong user or password + """ + data = request.json + + if len(data["password"]) < 8: + return dict(message='Password not long enough'), 404 + + user = get_user_by_id("local:" + current_user["email"]) + + if user is None: + return dict(message='User does not exist.'), 404 + + try: + if not checkpw(data["passOld"].encode('utf-8'), user["password"].encode('utf-8')): + return dict(message='Bad password.'), 404 + except Exception as e: + return { + "message": "Something went wrong!", + "error": str(e), + "data": None + }, 500 + + result = set_new_password(get_hashed_password(data["password"]), "local:" + current_user["email"]) + if result is not None: + # exception occurred + return dict(message="Password change failed. Please, try again and if the problem persists, contact NERD administrator."), 400 + else: + return dict(message="Password has been successfully changed!"), 200 + + + + +@api_v2.route('/refreshToken', methods=['POST']) +def refreshToken(): + """Resresh login token + --- + consumes: + - application/json + parameters: + - in: body + name: body + description: Request body for endpoint + required: true + schema: + type: object + properties: + token: + type: string + description: refresh token + example: "ey..." + responses: + 200: + description: Token refreshed + 400: + description: wrong format + 403: + description: Invalid Authentication token + 404: + description: Email address not verified + """ + data = request.json + try: + out = jwt.decode(data["token"], config.get("secret_key"), algorithms=["HS256"]) + except Exception as e: + return { + "message": "Something went wrong", + "data": None, + "error": "wrong format" + }, 400 + # check if anything changed in between token gens + current_user = get_user_by_id(out["user"]["id"]) + if current_user is None: + return { + "message": "Invalid Authentication token!", + "data": None, + "error": "Unauthorized" + }, 403 + if not "registered" in current_user["groups"]: + return { + "message": "Email address not verified", + "data": None, + }, 404 + out2 = out + out['exp'] = datetime.utcnow()+timedelta(hours=4) + out2['exp'] = datetime.utcnow()+timedelta(hours=72) + accessToken = jwt.encode(out, config.get('secret_key'), algorithm="HS256") + refreshToken = jwt.encode(out2, config.get('secret_key'), algorithm="HS256") + return Response(json.dumps([accessToken, refreshToken], default=str), 200, mimetype='application/json') + +######################################################## +# ADMIN ENDPOINTS # +######################################################## + +@api_v2.route('/users', methods=['GET']) +@token_required +def users_info(current_user): + """Gets all users + --- + security: + - OAuth2: [admin] + responses: + 200: + description: Users + 500: + description: Internal server error + """ + # check if user accessing this endpoint is admin + if 'admin' not in current_user["groups"]: + return dict(message='Access denied.'), 500 + + try: + users = get_users_admin() + except Exception as e: + return { + "message": "Something went wrong!", + "error": str(e), + "data": None + }, 500 + + return jsonify(users), 200 + + +@api_v2.route('/roles', methods=['PUT']) +@token_required +def change_roles(current_user): + """Change of user roles + --- + security: + - OAuth2: [admin] + consumes: + - application/json + parameters: + - in: body + name: body + description: Request body for endpoint + required: true + schema: + type: object + properties: + id: + type: string + description: user email + example: "local:user@email.com" + roles: + type: array + description: array of user roles + example: ["registered", "admin"] + responses: + 200: + description: Roles changed successfully + 500: + description: Internal server error + """ + # check if user accessing this endpoint is admin + if 'admin' not in current_user["groups"]: + return dict(message='Access denied.'), 500 + + data = request.json + # user whose roles are to be changed + ide = data["id"] + # new array of roles + roles = data["roles"] + try: + out = set_new_roles(ide, roles) + except Exception as e: + return { + "message": "Something went wrong!", + "error": str(e), + "data": None + }, 500 + if not out: + return { + "message": "Error while saving data to DB.", + "error": "DB Error", + "data": None + }, 500 + + return dict(message="Roles chaged successfully !"), 200 + +@api_v2.route('/delete_user/', methods=['DELETE']) +@token_required +def delete(current_user, ide): + """Deletes user + --- + security: + - OAuth2: [admin] + parameters: + - name: ide + in: path + type: string + required: true + description: ID of user to delete + responses: + 200: + description: User deleted successfully + 500: + description: Internal server error + """ + # check if user accessing this endpoint is admin + if 'admin' not in current_user["groups"]: + return dict(message='Access denied.'), 500 + + try: + delete_user(ide) + except Exception as e: + return { + "message": "Something went wrong!", + "error": str(e), + "data": None + }, 500 + + return dict(message="User deleted successfully!"), 200 + +@api_v2.route('/add-user', methods=['POST']) +@token_required +def add_user(current_user): + """Add new user + --- + security: + - OAuth2: [admin] + consumes: + - application/json + parameters: + - in: body + name: body + description: Request body for endpoint + required: true + schema: + type: object + properties: + email: + type: string + description: user email + example: "local:user@email.com" + password: + type: string + description: users passroed + example: "password123" + organization: + type: string + description: users organization + example: "org" + roles: + type: array + description: array of user roles + example: ["registered", "admin"] + verify: + type: boolean + description: mark email as verified + example: true + responses: + 200: + description: User added succesfully + 400: + description: Error while creating user + 500: + description: Internal server error + """ + # check if user accessing this endpoint is admin + if 'admin' not in current_user["groups"]: + return dict(message='Access denied.'), 500 + data = request.json + hashed_password = get_hashed_password(data["password"]) + res = create_user(data["email"], hashed_password, "local", data["name"], data["organization"], data["roles"]) + if isinstance(res, Exception): + if res.args[0].startswith("duplicate key") and "Key (id)" in res.args[0]: #TODO? neexistuje lep�� zp�sob kontroly typu cyhby? Pr� na to speci�ln� typ Exc asi nen� + return dict(message=f"User with email address {data['email']} already exists!"), 400 + else: + return dict(message=f"ERROR in create_user(): Something has failed during user creation process: {res.args}"), 400 + if data["verify"]: + just_verify_user_by_id("local:" + data["email"]) + else: + out = {} + out['user'] = { + 'email': data['email'], + } + out['exp'] = datetime.utcnow()+timedelta(hours=24) + encoded_jwt = jwt.encode(out, config.get('secret_key'), algorithm="HS256") + msg = Message(subject="[NERD] Account cerated", + recipients=[data["email"]], + reply_to=current_app.config.get('MAIL_DEFAULT_SENDER'), + ) + msg.html = f"

NERD

Network Entity Reputation Database

Email address verification

Dear user {data['email']},

you can activate your NERD account by clicking the link below (the link is valid for 24 hours):

{config.get('email_web')}/verify


NERD administrator

" + mailer.send(msg) + set_verification_email_sent(datetime.utcnow(), data["email"]) + + return dict(message=f"Account created for {data['email']}!"), 200 + + +######################################################## +# EXTERNAL IDENTITY PROVIDERS # +######################################################## + + +@api_v2.route('/oauth/google', methods=['GET']) +def oauth_google(): + code = request.args.get("code") + # Set the token endpoint URL + token_url = "https://oauth2.googleapis.com/token" + + # Set the parameters for the POST request + client_id = "864125509519-s4prljrk97usreg167i7de2rkppfa4re.apps.googleusercontent.com" + client_secret = "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" + redirect_uri = APP_BASE_URL + "/nerd/api/v2/oauth/google" + grant_type = "authorization_code" + + # Make the POST request to exchange the authorization code for an access token + response = requests.post(token_url, data={ + "code": code, + "client_id": client_id, + "client_secret": client_secret, + "redirect_uri": redirect_uri, + "grant_type": grant_type + }) + + # Get the access token and user info from the response + access_token = response.json()["access_token"] + token_type = response.json()["token_type"] + expires_in = response.json()["expires_in"] + refresh_token = response.json()["refresh_token"] + + # Use the access token to get user info + user_info_url = "https://www.googleapis.com/oauth2/v3/userinfo" + headers = {"Authorization": f"{token_type} {access_token}"} + user_info_response = requests.get(user_info_url, headers=headers) + + # Get the user info from the response + user_info = user_info_response.json() + + created = "false" + + # Now you can check if the user's email is in your local DB + if get_user_by_id("google:" + user_info["email"]) is None: + res = create_user(user_info["email"], None, "google", "NoName", "NoOrg") + if isinstance(res, Exception): + if res.args[0].startswith("duplicate key") and "Key (id)" in res.args[0]: #TODO? neexistuje lep�� zp�sob kontroly typu cyhby? Pr� na to speci�ln� typ Exc asi nen� + return dict(message=f"User with email address {data['email']} already exists! You can either log in or try to " + f"reset your password in log-in section."), 400 + else: + return dict(message=f"ERROR in register_user(): Something has failed during registration process: {res.args}"), 400 + else: + verify_user("google:" + user_info["email"]) + created = "true" + out = {} + out['user'] = { + 'login_type': 'google', + 'id': 'google:' + user_info["email"], + 'email': user_info["email"], + } + out['exp'] = datetime.utcnow()+timedelta(hours=4) + token = jwt.encode(out, config.get('secret_key'), algorithm="HS256") + out2 = {} + out2['user'] = { + 'login_type': 'google', + 'id': 'google:' + user_info["email"], + 'email': user_info["email"], + } + out2['exp'] = datetime.utcnow()+timedelta(hours=72) + refreshToken = jwt.encode(out2, config.get('secret_key'), algorithm="HS256") + set_last_login(datetime.utcnow(), 'google:' + user_info["email"]) + return redirect(APP_BASE_URL + "/nerd2/auth?token=" + token + "&refreshToken=" + refreshToken + "&created=" + created, code=302) + + +@api_v2.route('/oauth/twitter/url', methods=['GET']) +def oauth_twitter_access(): + #create an object of OAuth1Session + request_token = OAuth1Session(client_key="pOVnN86c7LgVcPhtnbc2dpmpu", client_secret="cRacpbtBs87wx3uEHQ6aE04SmyS0JpwlZG8W5TfL9WElMp7hVq") + # twitter endpoint to get request token + url = 'https://api.twitter.com/oauth/request_token' + # get request_token_key, request_token_secret and other details + data = request_token.get(url) + # split the string to get relevant data + data_token = str.split(data.text, '&') + ro_key = str.split(data_token[0], '=') + ro_secret = str.split(data_token[1], '=') + resource_owner_key = ro_key[1] + resource_owner_secret = ro_secret[1] + return redirect("https://api.twitter.com/oauth/authenticate?oauth_token=" + resource_owner_key) + +@api_v2.route('/oauth/twitter', methods=['GET']) +def oauth_twitter(): + token = request.args.get("oauth_token") + verifier = request.args.get("oauth_verifier") + oauth_token = OAuth1Session(client_key="pOVnN86c7LgVcPhtnbc2dpmpu", client_secret="cRacpbtBs87wx3uEHQ6aE04SmyS0JpwlZG8W5TfL9WElMp7hVq") + url = 'https://api.twitter.com/oauth/access_token' + data = {"oauth_verifier": verifier, "oauth_token": token} + + access_token_data = oauth_token.post(url, data=data) + access_token_list = str.split(access_token_data.text, '&') + + access_token_key = str.split(access_token_list[0], '=') + access_token_secret = str.split(access_token_list[1], '=') + access_token_name = str.split(access_token_list[3], '=') + access_token_id = str.split(access_token_list[2], '=') + key = access_token_key[1] + secret = access_token_secret[1] + oauth_user = OAuth1Session(client_key="pOVnN86c7LgVcPhtnbc2dpmpu", client_secret="cRacpbtBs87wx3uEHQ6aE04SmyS0JpwlZG8W5TfL9WElMp7hVq", + resource_owner_key=key, + resource_owner_secret=secret) + url_user = 'https://api.twitter.com/1.1/account/verify_credentials.json' + params = {"include_email": 'true'} + user_data = oauth_user.get(url_user, params=params) + user_info = user_data.json() + created = "false" + if get_user_by_id('twitter:' + user_info["email"]) is None: + res = create_user(user_info["email"], None, "twitter", "NoName", "NoOrg") + if isinstance(res, Exception): + if res.args[0].startswith("duplicate key") and "Key (id)" in res.args[0]: + return dict(message=f"User with email address {data['email']} already exists! You can either log in or try to " + f"reset your password in log-in section."), 400 + else: + return dict(message=f"ERROR in register_user(): Something has failed during registration process: {res.args}"), 400 + else: + verify_user("twitter:" + user_info["email"]) + created = "true" + out = {} + out['user'] = { + 'login_type': 'twitter', + 'id': 'twitter:' + user_info["email"], + 'email': user_info["email"], + } + out['exp'] = datetime.utcnow()+timedelta(hours=4) + token = jwt.encode(out, config.get('secret_key'), algorithm="HS256") + out2 = {} + out2['user'] = { + 'login_type': 'twitter', + 'id': 'twitter:' + user_info["email"], + 'email': user_info["email"], + } + out2['exp'] = datetime.utcnow()+timedelta(hours=72) + refreshToken = jwt.encode(out2, config.get('secret_key'), algorithm="HS256") + set_last_login(datetime.utcnow(), 'twitter:' + user_info["email"]) + return redirect(APP_BASE_URL + "/nerd2/auth?token=" + token + "&refreshToken=" + refreshToken + "&created=" + created, code=302) + +@api_v2.route('/oauth/github', methods=['GET']) +def oauth_github(): + code = request.args.get("code") + # Set the token endpoint URL + token_url = "https://github.com/login/oauth/access_token" + + # Set the parameters for the POST request + client_id = "b13789eb1250d5e77992" + client_secret = "c9de015e91e288f96973b7aa95406c44284687b4" + + # Make the POST request to exchange the authorization code for an access token + response = requests.post(token_url, data={ + "code": code, + "client_id": client_id, + "client_secret": client_secret + }) + + access_token = 'token ' + response.text.split("&")[0].split("=")[1] + url = 'https://api.github.com/user/emails' + headers = {"Authorization": access_token} + + resp = requests.get(url=url, headers=headers) + + userData = resp.json() + for email in userData: + if email["primary"]: + user_email = email["email"] + break + created = "false" + + if get_user_by_id('github:' + user_email) is None: + res = create_user(user_email, None, "github", "NoName", "NoOrg") + if isinstance(res, Exception): + if res.args[0].startswith("duplicate key") and "Key (id)" in res.args[0]: + return dict(message=f"User with email address {data['email']} already exists! You can either log in or try to " + f"reset your password in log-in section."), 400 + else: + return dict(message=f"ERROR in register_user(): Something has failed during registration process: {res.args}"), 400 + else: + verify_user("github:" + user_email) + created = "true" + out = {} + out['user'] = { + 'login_type': 'github', + 'id': 'github:' + user_email, + 'email': user_email, + } + out['exp'] = datetime.utcnow()+timedelta(hours=4) + token = jwt.encode(out, config.get('secret_key'), algorithm="HS256") + out2 = {} + out2['user'] = { + 'login_type': 'github', + 'id': 'github:' + user_email, + 'email': user_email, + } + out2['exp'] = datetime.utcnow()+timedelta(hours=72) + refreshToken = jwt.encode(out2, config.get('secret_key'), algorithm="HS256") + set_last_login(datetime.utcnow(), 'github:' + user_email) + return redirect(APP_BASE_URL + "/nerd2/auth?token=" + token + "&refreshToken=" + refreshToken + "&created=" + created, code=302) + + +def get_well_known_metadata(): + response = requests.get("https://login.cesnet.cz/oidc/.well-known/openid-configuration") + response.raise_for_status() + return response.json() + + +def get_oauth2_session(**kwargs): + oauth2_session = OAuth2Session("029639e5-09f4-461d-859a-7b06aa29d61e", + scope=["profile", "email", "openid"], + redirect_uri=APP_BASE_URL + "/nerd/api/v2/oauth/eduid", + **kwargs) + return oauth2_session + +@api_v2.route("/oauth/edugain/url") +def login_edugain(): + well_known_metadata = get_well_known_metadata() + oauth2_session = get_oauth2_session() + authorization_url, state = oauth2_session.authorization_url(well_known_metadata["authorization_endpoint"]) + session["oauth_state"] = state + return redirect(authorization_url) + +@api_v2.route("/oauth/eduid") +def edu_id_callback(): + well_known_metadata = get_well_known_metadata() + oauth2_session = get_oauth2_session(state=request.args["state"]) + oauth_token = oauth2_session.fetch_token(well_known_metadata["token_endpoint"], + client_secret="73743e93-9560-4d5b-983f-33911afce589cb6c980a-4baa-47e0-8317-c11f786b4254", + code=request.args["code"])["id_token"] + resp = oauth2_session.get(well_known_metadata["userinfo_endpoint"]) + data = resp.json() + + # if the user just created a profile + created = "false" + + # EduGain user not in DB yet + eduUser = get_user_by_id('edugain:' + data["email"]) + + if eduUser is None: + # check if user is in DB from old NERD system + shiUser = get_user_by_id('shibboleth:' + data["email"]) + if shiUser is None: + # new user create profile + res = create_user(data["email"], None, "edugain", "NoName", "NoOrg") + if isinstance(res, Exception): + if res.args[0].startswith("duplicate key") and "Key (id)" in res.args[0]: + return dict(message=f"User with email address {data['email']} already exists! You can either log in or try to " + f"reset your password in log-in section."), 400 + else: + return dict(message=f"ERROR in register_user(): Something has failed during registration process: {res.args}"), 400 + else: + verify_user("edugain:" + data["email"]) + created = "true" + else: + # user from old NERD merge profile + res = create_user(data["email"], None, "edugain", shiUser["name"], shiUser["org"]) + if isinstance(res, Exception): + if res.args[0].startswith("duplicate key") and "Key (id)" in res.args[0]: + return dict(message=f"User with email address {data['email']} already exists! You can either log in or try to " + f"reset your password in log-in section."), 400 + else: + return dict(message=f"ERROR in register_user(): Something has failed during registration process: {res.args}"), 400 + else: + verify_user("edugain:" + data["email"]) + set_api_v1_token("edugain:" + data["email"], shiUser["api_token"]) + delete_user('shibboleth:' + data["email"]) + created = "true" + + out = {} + out['user'] = { + 'login_type': 'edugain', + 'id': 'edugain:' + data["email"], + 'email': data["email"], + } + out['exp'] = datetime.utcnow()+timedelta(hours=4) + token = jwt.encode(out, config.get('secret_key'), algorithm="HS256") + out2 = {} + out2['user'] = { + 'login_type': 'edugain', + 'id': 'edugain:' + data["email"], + 'email': data["email"], + } + out2['exp'] = datetime.utcnow()+timedelta(hours=72) + refreshToken = jwt.encode(out2, config.get('secret_key'), algorithm="HS256") + set_last_login(datetime.utcnow(), 'edugain:' + data["email"]) + return redirect(APP_BASE_URL + "/nerd2/auth?token=" + token + "&refreshToken=" + refreshToken + "&created=" + created, code=302) \ No newline at end of file diff --git a/NERDweb/api_v2_swag/login.yml b/NERDweb/api_v2_swag/login.yml new file mode 100644 index 00000000..579aa574 --- /dev/null +++ b/NERDweb/api_v2_swag/login.yml @@ -0,0 +1,29 @@ +User local login + --- + consumes: + - application/json + parameters: + - in: body + name: body + description: Request body for endpoint + required: true + schema: + type: object + properties: + email: + type: string + description: Email of the local user + example: "user@rmail.com" + password: + type: string + description: Password of the local user + example: "password123" + responses: + 200: + description: Login successful + 400: + description: Bad password + 404: + description: User does not exist + 500: + description: Inetrnal server error \ No newline at end of file diff --git a/NERDweb/api_v2_swag/register.yml b/NERDweb/api_v2_swag/register.yml new file mode 100644 index 00000000..1b1164c0 --- /dev/null +++ b/NERDweb/api_v2_swag/register.yml @@ -0,0 +1,37 @@ +User local registration + --- + consumes: + - application/json + parameters: + - in: body + name: body + description: Request body for endpoint + required: true + schema: + type: object + properties: + email: + type: string + description: Email of the local user + example: "user@rmail.com" + password: + type: string + description: Password of the local user + example: "password123" + organization: + type: string + enum: + - Academic + - Research + - Security + description: Organization of the local user + example: "user@rmail.com" + name: + type: string + description: Name of the local user + example: "noName for v2" + responses: + 200: + description: Registration successful + 400: + description: Error during registration \ No newline at end of file diff --git a/NERDweb/api_v2_swag/search.yml b/NERDweb/api_v2_swag/search.yml new file mode 100644 index 00000000..3a8ce5e5 --- /dev/null +++ b/NERDweb/api_v2_swag/search.yml @@ -0,0 +1,161 @@ +Basic info for a single IP address + --- + consumes: + - application/json + parameters: + - in: body + name: body + description: Request body for endpoint + required: true + schema: + type: object + properties: + subnet: + nullable: true + type: ['null', array] + items: + type: string + description: IPv4 prefix/subnet in CIDR format + example: ["146.88.240.4", "192.168.1.0/24"] + hostname: + nullable: true + type: ['null', array] + items: + type: string + description: Hostname suffix + example: ["security.criminalip.com"] + country: + nullable: true + type: ['null', array] + items: + type: string + description: IP address country of origin + example: ["US"] + asn: + nullable: true + type: ['null', array] + description: Autonomous system number + example: AS202425 + cat: + nullable: true + type: ['null', array] + description: Event category - Warden + example: AnomalyBehaviour + blacklist: + nullable: true + type: ['null', array] + description: Appearance on blacklist(s) + example: AbuseIPDB + source: + nullable: true + type: ['null', array] + description: IP address malicious intent detected by source + example: warden + tag: + nullable: true + type: ['null', array] + description: Additional tags + example: attemptexploit + sort: + nullable: true + type: ['null', string] + enum: + - ip + - ts_added + - last_activity + - rep + description: allowed - ip, rep, ts_added and last_activity + example: ip + order: + nullable: true + type: ['null', string] + enum: + - asc + - desc + description: Order ascending or descending + example: desc + limit: + nullable: true + type: ['null', number] + enum: + - 5 + - 10 + - 20 + - 50 + description: results per page + example: 20 + asn_op: + nullable: true + type: ['null', string] + enum: + - AND + - OR + description: AND / OR logical operator that parses through given asn values + example: or + source_op: + nullable: true + type: ['null', string] + enum: + - AND + - OR + description: AND / OR logical operator that parses through given source values + example: or + cat_op: + nullable: true + type: ['null', string] + enum: + - AND + - OR + description: AND / OR logical operator that parses through given category values + example: or + bl_op: + nullable: true + type: ['null', string] + enum: + - AND + - OR + description: AND / OR logical operator that parses through given blacklist values + example: or + tag_op: + nullable: true + type: ['null', string] + enum: + - AND + - OR + description: AND / OR logical operator that parses through given tag values + example: or + whitelisted: + nullable: true + type: boolean + description: Hide or show whitelisted IPs + example: true + page: + nullable: true + type: number + description: Page number when listing muliple results + example: 1 + responses: + '200': + description: OK + schema: + type: object + properties: + message: + type: string + example: Endpoint response message + '400': + description: Bad Request + schema: + type: object + properties: + message: + type: string + example: Endpoint response message + '500': + description: Internal Server Error + schema: + type: object + properties: + message: + type: string + example: Endpoint response message \ No newline at end of file diff --git a/NERDweb/auth.py b/NERDweb/auth.py new file mode 100644 index 00000000..00570bb1 --- /dev/null +++ b/NERDweb/auth.py @@ -0,0 +1,51 @@ +from functools import wraps +import jwt +from flask import request, abort +from flask import current_app +from userdb import get_user_by_id +import os +import common.config + + +config = common.config.read_config(os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "/etc/nerd/nerdweb.yml"))) + +def token_required(f): + @wraps(f) + def decorated(*args, **kwargs): + token = None + if "Authorization" in request.headers: + token = request.headers["Authorization"] + if not token: + return { + "message": "Authentication Token is missing!", + "data": None, + "error": "Unauthorized" + }, 401 + try: + data=jwt.decode(token, config.get("secret_key"), algorithms=["HS256"]) + + if data["user"]["login_type"] == "devel": + return data + current_user=get_user_by_id(data["user"]["id"]) + if current_user is None: + return { + "message": "Invalid Authentication token!", + "data": None, + "error": "Unauthorized" + }, 401 + if not "registered" in current_user["groups"]: + return { + "message": "Email address not verified", + "data": None, + }, 403 + except Exception as e: + return { + "message": "Something went wrong", + "data": str(e), + "error": "AuthToken wrong" + }, 401 + + return f(current_user, *args, **kwargs) + + return decorated + diff --git a/NERDweb/com.py b/NERDweb/com.py new file mode 100644 index 00000000..7f9a3ca1 --- /dev/null +++ b/NERDweb/com.py @@ -0,0 +1,407 @@ +import os +import sys +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))) +import common.config +from flask_mail import Mail +from flask import Flask, Response +from flask_pymongo import PyMongo +from event_count_logger import EventCountLogger, EventGroup, DummyEventGroup +from flask_wtf import FlaskForm +from wtforms import validators, StringField, TextAreaField, FloatField, IntegerField, BooleanField, HiddenField, SelectField, SelectMultipleField, PasswordField +from common.utils import ipstr2int, int2ipstr, parse_rfc_time +from datetime import datetime, timedelta, timezone + +DEFAULT_CONFIG_FILE = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "/etc/nerd/nerdweb.yml")) + +# TODO parse arguments using ArgParse +if len(sys.argv) >= 2: + cfg_file = sys.argv[1] +else: + cfg_file = DEFAULT_CONFIG_FILE +cfg_dir = os.path.dirname(os.path.abspath(cfg_file)) + +# Read web-specific config (nerdweb.cfg) +config = common.config.read_config(cfg_file) + +# Read EventCountLogger config (to separate dict) and initialize loggers +ecl_cfg_filename = config.get('event_logging_config', None) +if ecl_cfg_filename: + # Load config + config_ecl = common.config.read_config(os.path.join(cfg_dir, ecl_cfg_filename)) + # Initialize EventCountLogger + ecl = EventCountLogger(config_ecl.get('groups'), config_ecl.get('redis', {})) + # Get instances of EventGroups (if specified in configuration, otherwise, DummyEventGroup is used, so logging is no-op) + # (it's recommended to enable local counters for both groups for better performance) + log_ep = ecl.get_group('web_endpoints') or DummyEventGroup() # log access to individual endpoints + log_err = ecl.get_group('web_errors') or DummyEventGroup() # log error replies +else: + print("WARNING: nerd_main: Path to event logging config ('event_logging_config' key) not specified, EventCountLogger disabled.") + log_ep = DummyEventGroup() + log_err = DummyEventGroup() + +app = Flask(__name__) + +app.secret_key = config.get('secret_key') + +app.jinja_env.trim_blocks = True +app.jinja_env.lstrip_blocks = True + +# Configuration of PyMongo +mongo_dbname = config.get('mongodb.dbname', 'nerd') +mongo_host = config.get('mongodb.host', 'localhost:27017') +if isinstance(mongo_host, list): + mongo_host = ','.join(mongo_host) +mongo_uri = "mongodb://{}/{}".format(mongo_host, mongo_dbname) +mongo_rs = config.get('mongodb.rs', None) +if mongo_rs: + mongo_uri += '?replicaSet='+mongo_rs +app.config['MONGO_URI'] = mongo_uri +print("MongoDB: Connecting to: {}".format(mongo_uri)) + +mongo = PyMongo(app) + +# Configuration of MAIL extension +app.config['MAIL_SERVER'] = config.get('mail.server', 'localhost') +app.config['MAIL_PORT'] = config.get('mail.port', '25') +app.config['MAIL_USE_TLS'] = config.get('mail.tls', False) +app.config['MAIL_USE_SSL'] = config.get('mail.ssl', False) +app.config['MAIL_USERNAME'] = config.get('mail.username', None) +app.config['MAIL_PASSWORD'] = config.get('mail.password', None) +app.config['MAIL_DEFAULT_SENDER'] = config.get('mail.sender', 'NERD ') + +mailer = Mail(app) + +def validator_optional(form, field): + if not field.data: + field.errors[:] = [] + raise validators.StopValidation() + +def strip_whitespace(s): + if isinstance(s, str): + s = s.strip() + return s + + +def clean_secret_data(data): + """Remove all keys starting with '_' (except '_id') from dict.""" + if data is not None: + for i in list(data): + if i.startswith("_") and i != "_id": + del data[i] + return data + +def attach_whois_data(ipinfo, full): + """ + Attach records of related entities to given IP record. + + If full==True, attach full records of BGP prefix, ASNs, IP block, Org entities (as 'bgppref, 'asn', 'ipblock' and 'org' keys), + otherwise only attach list of ASN numbers (as 'asn' key). + """ + if not full: + # Only attach ASN number(s) + if 'bgppref' in ipinfo: + bgppref_rec = mongo.db.bgppref.find_one({'_id': ipinfo['bgppref']}, {'asn': 1}) + if bgppref_rec is None: + print("ERROR: Can't find BGP prefix '{}' in database (trying to enrich IP {})".format(ipinfo['bgppref'], ipinfo['_id'])) + return + if 'asn' in bgppref_rec: + ipinfo['asn'] = bgppref_rec['asn'] + return + + # Full - attach full records of related BGP prefix, ASNs, IP block, Org + # IP->BGPpref + if 'bgppref' in ipinfo: + bgppref_rec = clean_secret_data(mongo.db.bgppref.find_one({'_id': ipinfo['bgppref']})) + if bgppref_rec is None: + print("ERROR: Can't find BGP prefix '{}' in database (trying to enrich IP {})".format(ipinfo['bgppref'], ipinfo['_id'])) + else: + # BGPpref->ASN(s) + asn_list = [] + for asn in bgppref_rec['asn']: + asn_rec = clean_secret_data(mongo.db.asn.find_one({'_id': asn})) + if asn_rec is None: + print("ERROR: Can't find ASN '{}' in database (trying to enrich IP {}, bgppref {})".format(asn, ipinfo['_id'], bgppref_rec['_id'])) + else: + # ASN->Org + if 'org' in asn_rec: + org_rec = clean_secret_data(mongo.db.org.find_one({'_id': asn_rec['org']})) + if org_rec is None: + print( + "ERROR: Can't find Org '{}' in database (trying to enrich IP {}, bgppref {}, ASN {})".format( + asn_rec['org'], ipinfo['_id'], bgppref_rec['_id'], asn)) + else: + conv_dates(org_rec) + asn_rec['org'] = org_rec + + del asn_rec['bgppref'] + conv_dates(asn_rec) + asn_list.append(asn_rec) + + del bgppref_rec['asn'] + conv_dates(bgppref_rec) + ipinfo['bgppref'] = bgppref_rec + ipinfo['asn'] = asn_list + + # IP->ipblock + if 'ipblock' in ipinfo: + ipblock_rec = (mongo.db.ipblock.find_one({'_id': ipinfo['ipblock']})) + if ipblock_rec is None: + print("ERROR: Can't find IP block '{}' in database (trying to enrich IP {})".format(ipinfo['ipblock'], + ipinfo['_id'])) + else: + # ipblock->org + if "org" in ipblock_rec: + org_rec = clean_secret_data(mongo.db.org.find_one({'_id': ipblock_rec['org']})) + if org_rec is None: + print("ERROR: Can't find Org '{}' in database (trying to enrich IP {}, ipblock '{}')".format( + ipblock_rec['org'], ipinfo['_id'], ipblock_rec['_id'])) + else: + conv_dates(org_rec) + ipblock_rec['org'] = org_rec + + conv_dates(ipblock_rec) + ipinfo['ipblock'] = ipblock_rec + +def conv_dates(rec): + """Convert datetimes in a record to YYYY-MM-DDTMM:HH:SS string""" + for key in ('ts_added', 'ts_last_update'): + if key in rec and isinstance(rec[key], datetime): + rec[key] = rec[key].strftime("%Y-%m-%dT%H:%M:%S") + +class SingleIPForm(FlaskForm): + ip = StringField('IP address', [validator_optional, validators.IPAddress(message="Invalid IPv4 address")], filters=[strip_whitespace]) + +def get_ip_info(ipaddr, full): + data = { + 'err_n': 400, + 'error': "No IP address specified", + 'ip': ipaddr + } + + if not ipaddr: + log_err.log('400_bad_request') + return False, Response(json.dumps(data), 400, mimetype='application/json') + + form = SingleIPForm(ip=ipaddr) + if not form.validate(): + log_err.log('400_bad_request') + data['error'] = "Bad IP address" + return False, Response(json.dumps(data), 400, mimetype='application/json') + + ipint = ipstr2int(form.ip.data) # Convert string IP to int + + if full: + ipinfo = mongo.db.ip.find_one({'_id': ipint}) + else: + ipinfo = mongo.db.ip.find_one({'_id': ipint}, + {'rep': 1, 'fmp': 1, 'hostname': 1, 'bgppref': 1, 'ipblock': 1, 'geo': 1, 'bl': 1, + 'tags': 1}) + if not ipinfo: + log_err.log('404_api_ip_not_found') + data['err_n'] = 404 + data['error'] = "IP address not found" + return False, Response(json.dumps(data), 404, mimetype='application/json') + + ipinfo['_id'] = int2ipstr(ipinfo['_id']) # Convert int IP to string + + attach_whois_data(ipinfo, full) + return True, ipinfo + +# ***** NERD API BasicInfo - helper funcs ***** +def get_basic_info_dic(val): + geo_d = {} + if 'geo' in val.keys(): + geo_d['ctry'] = val['geo'].get('ctry', "unknown") + + bl_l = [] + for l in val.get('bl', []): + bl_l.append(l['n']) # TODO: shouldn't there be a check for v=1? + + tags_l = [] + for l in val.get('tags', []): + d = { + 'n': l, + 'c': val['tags'][l]['confidence'] + } + + tags_l.append(d) + + data = { + 'ip': val['_id'], + 'rep': val.get('rep', 0.0), + 'fmp': val.get('fmp', {'general': 0.0}), + 'hostname': (val.get('hostname', '') or '')[::-1], + 'ipblock': val.get('ipblock', ''), + 'bgppref': val.get('bgppref', ''), + 'asn': val.get('asn', []), + 'geo': geo_d, + 'bl': bl_l, + 'tags': tags_l + } + + return data + + +def get_basic_info_dic_short(val): + # only 'rep' and 'tags' fields + tags_l = [] + for l in val.get('tags', []): + d = { + 'n': l, + 'c': val['tags'][l]['confidence'] + } + tags_l.append(d) + + data = { + 'ip': val['_id'], + 'rep': val.get('rep', 0.0), + 'tags': tags_l + } + return data + +def get_basic_info_dic_v2(val): + geo_d = {} + if 'geo' in val.keys(): + geo_d['ctry'] = val['geo'].get('ctry', "unknown") + + bl_l = [] + for l in val.get('bl', []): + bl_l.append(l['n']) # TODO: shouldn't there be a check for v=1? + + tags_l = [] + for l in val.get('tags', []): + d = { + 'n' : l, + 'c' : val['tags'][l]['confidence'] + } + + tags_l.append(d) + + data = { + 'ip' : val['_id'], + 'rep' : val.get('rep', 0.0), + 'fmp' : val.get('fmp', {'general': 0.0}), + 'hostname' : (val.get('hostname', '') or '')[::-1], + 'ipblock' : val.get('ipblock', ''), + 'bgppref' : val.get('bgppref', ''), + 'asn' : val.get('asn',[]), + 'geo' : geo_d, + 'bl' : bl_l, + 'tags' : tags_l, + 'ts_last_update' : val.get('ts_last_update', ''), + 'ts_added' : val.get('ts_added', ''), + } + + return data + + +def find_ip_data(query, skip_n, limit): + return mongo.db.ip.find(query).skip(skip_n).limit(limit) + +def ip_to_warden_data(ipaddr): + ipint = ipstr2int(ipaddr) + ipinfo = mongo.db.ip.aggregate(pipeline = [ + { '$match': { '_id': ipint } }, + { '$unwind': '$events' }, + { '$group': { + '_id': { + 'date': '$events.date', + 'cat': '$events.cat' + }, + 'n_sum': { '$sum': '$events.n' }, + 'conns_sum': { '$sum': '$events.conns' }, + } + }, + { '$group': { + '_id': '$_id.date', + 'categories': { + "$push": { + "k": "$_id.cat", + "v": { + "n_sum": "$n_sum", + "conns_sum": "$conns_sum", + "nodes": "$nodes" + } + } + } + } + }, + { '$project': { + '_id': 0, + 'date': '$_id', + "categories": {"$arrayToObject": "$categories"} + } + }, + { '$sort': { 'date': 1 } } + ]) + + ipinfo_list = [doc for doc in ipinfo] + #ipinfo_list['_id'] = int2ipstr(ipinfo_list['_id']) + return ipinfo_list + + +def create_query_v2(data): + # Prepare 'find' part of the query + queries = [] + if data is None: + return None + + if "subnet" in data and data["subnet"] is not None and len(data["subnet"]) != 0: + subqueries = [] + for subnet in data["subnet"]: + subnet = ipaddress.IPv4Network(subnet, strict=False) + subnet_start = int(subnet.network_address) # IP addresses are stored as int + subnet_end = int(subnet.broadcast_address) + subqueries.append( {'$and': [{'_id': {'$gte': subnet_start}}, {'_id': {'$lte': subnet_end}}]} ) + queries.append({'$or': subqueries}) + + if "hostname" in data and data["hostname"] is not None and len(data["hostname"]) != 0: + subqueries = [] + for hostname in data["hostname"]: + hn = hostname[::-1] # Hostnames are stored reversed in DB to allow search by suffix as a range search + hn_end = hn[:-1] + chr(ord(hn[-1])+1) + subqueries.append( {'$and': [{'hostname': {'$gte': hn}}, {'hostname': {'$lt': hn_end}}]} ) + queries.append({'$or': subqueries}) + + if "country" in data and data["country"] is not None and len(data["country"]) != 0: + queries.append( { '$or': [{'geo.ctry': c.upper() } for c in data["country"]]} ) + + if "asn" in data and data["asn"] is not None and len(data["asn"]) != 0 : + subqueries = [] + for asn in data["asn"]: + # ASN is not stored in IP records - get list of BGP prefixes of the ASN and filter by these + asn = int(asn.lstrip("ASas")) + asrec = mongo.db.asn.find_one({'_id': asn}) + if asrec and 'bgppref' in asrec: + subqueries.append( {'bgppref': {'$in': asrec['bgppref']}} ) + else: + subqueries.append( {'_id': {'$exists': False}} ) # ASN not in DB, add query which is always false to get no results + op = '$and' if (data["asn_op"] == "AND") else '$or' + queries.append({op: subqueries}) + + if "source" in data and data["source"] is not None and len(data["source"]) != 0: + op = '$and' if (data["source_op"] == "AND") else '$or' + queries.append( {op: [{'_ttl.' + s.lower(): {'$exists': True}} for s in data["source"]]} ) + + if "cat" in data and data["cat"] is not None and len(data["cat"]) != 0: + op = '$and' if (data["cat_op"] == "AND") else '$or' + queries.append( {op: [{'events.cat': cat} for cat in data["cat"]]} ) + + if "node" in data and data["node"] is not None and len(data["node"]) != 0: + op = '$and' if (data["node_op"] == "AND") else '$or' + queries.append( {op: [{'events.node': node} for node in data["node"]]} ) + + if "blacklist" in data and data["blacklist"] is not None and len(data["blacklist"]) != 0: + op = '$and' if (data["bl_op"] == "AND") else '$or' + array = [{('dbl' if t == 'd' else 'bl'): {'$elemMatch': {'n': id, 'v': 1}}} for t,_,id in map(lambda s: s.partition(':'), data["blacklist"])] + queries.append( {op: array} ) + + if "tag" in data and data["tag"] is not None and len(data["tag"]) != 0: + op = '$and' if (data["tag_op"] == "AND") else '$or' + queries.append( {op: [{'tags.'+ tag_id: {'$exists': True}} for tag_id in data["tag"]]} ) + + if "whitelisted" in data and data["whitelisted"]: + queries.append( {'tags.whitelist': {'$exists': False}} ) + + query = {'$and': queries} if queries else None + return query \ No newline at end of file diff --git a/NERDweb/nerd_main.py b/NERDweb/nerd_main.py index e9311ac6..28cfddc8 100644 --- a/NERDweb/nerd_main.py +++ b/NERDweb/nerd_main.py @@ -26,6 +26,8 @@ from pymisp import ExpandedPyMISP from ipaddress import IPv4Address, AddressValueError from event_count_logger import EventCountLogger, EventGroup, DummyEventGroup +# Open API requirements +from flasgger import Swagger # Add to path the "one directory above the current file location" sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))) @@ -39,6 +41,8 @@ import userdb import ratelimit from userdb import get_user_info, authenticate_with_token, generate_unique_token +from com import attach_whois_data, app, mongo, mailer, validator_optional, strip_whitespace, clean_secret_data, conv_dates, \ + get_ip_info, get_basic_info_dic, get_basic_info_dic_short, find_ip_data, ip_to_warden_data # ***** Load configuration ***** @@ -159,37 +163,37 @@ # **** Create and initialize Flask application ***** -app = Flask(__name__) +# app = Flask(__name__) -app.secret_key = config.get('secret_key') +# app.secret_key = config.get('secret_key') -app.jinja_env.trim_blocks = True -app.jinja_env.lstrip_blocks = True +# app.jinja_env.trim_blocks = True +# app.jinja_env.lstrip_blocks = True -# Configuration of PyMongo -mongo_dbname = config.get('mongodb.dbname', 'nerd') -mongo_host = config.get('mongodb.host', 'localhost:27017') -if isinstance(mongo_host, list): - mongo_host = ','.join(mongo_host) -mongo_uri = "mongodb://{}/{}".format(mongo_host, mongo_dbname) -mongo_rs = config.get('mongodb.rs', None) -if mongo_rs: - mongo_uri += '?replicaSet='+mongo_rs -app.config['MONGO_URI'] = mongo_uri -print("MongoDB: Connecting to: {}".format(mongo_uri)) +# # Configuration of PyMongo +# mongo_dbname = config.get('mongodb.dbname', 'nerd') +# mongo_host = config.get('mongodb.host', 'localhost:27017') +# if isinstance(mongo_host, list): +# mongo_host = ','.join(mongo_host) +# mongo_uri = "mongodb://{}/{}".format(mongo_host, mongo_dbname) +# mongo_rs = config.get('mongodb.rs', None) +# if mongo_rs: +# mongo_uri += '?replicaSet='+mongo_rs +# app.config['MONGO_URI'] = mongo_uri +# print("MongoDB: Connecting to: {}".format(mongo_uri)) -mongo = PyMongo(app) +# mongo = PyMongo(app) -# Configuration of MAIL extension -app.config['MAIL_SERVER'] = config.get('mail.server', 'localhost') -app.config['MAIL_PORT'] = config.get('mail.port', '25') -app.config['MAIL_USE_TLS'] = config.get('mail.tls', False) -app.config['MAIL_USE_SSL'] = config.get('mail.ssl', False) -app.config['MAIL_USERNAME'] = config.get('mail.username', None) -app.config['MAIL_PASSWORD'] = config.get('mail.password', None) -app.config['MAIL_DEFAULT_SENDER'] = config.get('mail.sender', 'NERD ') +# # Configuration of MAIL extension +# app.config['MAIL_SERVER'] = config.get('mail.server', 'localhost') +# app.config['MAIL_PORT'] = config.get('mail.port', '25') +# app.config['MAIL_USE_TLS'] = config.get('mail.tls', False) +# app.config['MAIL_USE_SSL'] = config.get('mail.ssl', False) +# app.config['MAIL_USERNAME'] = config.get('mail.username', None) +# app.config['MAIL_PASSWORD'] = config.get('mail.password', None) +# app.config['MAIL_DEFAULT_SENDER'] = config.get('mail.sender', 'NERD ') -mailer = Mail(app) +# mailer = Mail(app) # Disable CSRF protection globally (it's OK to send search requests from anywhere) # FIXME: This disables it completeley, it would be better to rather disable it @@ -320,16 +324,16 @@ def misp_get_cluster_count(event): # The original one uses field.raw_data, but we need field.data # (raw_data only contain data loaded from Form (GET/POST), not those passed # to Form constructor using obj/data/kwargs parameters) -def validator_optional(form, field): - if not field.data: - field.errors[:] = [] - raise validators.StopValidation() +# def validator_optional(form, field): +# if not field.data: +# field.errors[:] = [] +# raise validators.StopValidation() # Filter to strip whitespaces in string -def strip_whitespace(s): - if isinstance(s, str): - s = s.strip() - return s +# def strip_whitespace(s): +# if isinstance(s, str): +# s = s.strip() +# return s # ***** Auxiliary functions ***** @@ -1679,183 +1683,108 @@ def api_user_info(): return Response(json.dumps(data), 200, mimetype='application/json') -def get_ip_info(ipaddr, full): - data = { - 'err_n': 400, - 'error': "No IP address specified", - 'ip': ipaddr - } - - if not ipaddr: - log_err.log('400_bad_request') - return False, Response(json.dumps(data), 400, mimetype='application/json') - - form = SingleIPForm(ip=ipaddr) - if not form.validate(): - log_err.log('400_bad_request') - data['error'] = "Bad IP address" - return False, Response(json.dumps(data), 400, mimetype='application/json') - - ipint = ipstr2int(form.ip.data) # Convert string IP to int - - if full: - ipinfo = mongo.db.ip.find_one({'_id': ipint}) - else: - ipinfo = mongo.db.ip.find_one({'_id': ipint}, - {'rep': 1, 'fmp': 1, 'hostname': 1, 'bgppref': 1, 'ipblock': 1, 'geo': 1, 'bl': 1, - 'tags': 1}) - if not ipinfo: - log_err.log('404_api_ip_not_found') - data['err_n'] = 404 - data['error'] = "IP address not found" - return False, Response(json.dumps(data), 404, mimetype='application/json') - - ipinfo['_id'] = int2ipstr(ipinfo['_id']) # Convert int IP to string - - attach_whois_data(ipinfo, full) - return True, ipinfo - - -def conv_dates(rec): - """Convert datetimes in a record to YYYY-MM-DDTMM:HH:SS string""" - for key in ('ts_added', 'ts_last_update'): - if key in rec and isinstance(rec[key], datetime): - rec[key] = rec[key].strftime("%Y-%m-%dT%H:%M:%S") - - -def attach_whois_data(ipinfo, full): - """ - Attach records of related entities to given IP record. - - If full==True, attach full records of BGP prefix, ASNs, IP block, Org entities (as 'bgppref, 'asn', 'ipblock' and 'org' keys), - otherwise only attach list of ASN numbers (as 'asn' key). - """ - if not full: - # Only attach ASN number(s) - if 'bgppref' in ipinfo: - bgppref_rec = mongo.db.bgppref.find_one({'_id': ipinfo['bgppref']}, {'asn': 1}) - if bgppref_rec is None: - print("ERROR: Can't find BGP prefix '{}' in database (trying to enrich IP {})".format(ipinfo['bgppref'], ipinfo['_id'])) - return - if 'asn' in bgppref_rec: - ipinfo['asn'] = bgppref_rec['asn'] - return - - # Full - attach full records of related BGP prefix, ASNs, IP block, Org - # IP->BGPpref - if 'bgppref' in ipinfo: - bgppref_rec = clean_secret_data(mongo.db.bgppref.find_one({'_id': ipinfo['bgppref']})) - if bgppref_rec is None: - print("ERROR: Can't find BGP prefix '{}' in database (trying to enrich IP {})".format(ipinfo['bgppref'], ipinfo['_id'])) - else: - # BGPpref->ASN(s) - asn_list = [] - for asn in bgppref_rec['asn']: - asn_rec = clean_secret_data(mongo.db.asn.find_one({'_id': asn})) - if asn_rec is None: - print("ERROR: Can't find ASN '{}' in database (trying to enrich IP {}, bgppref {})".format(asn, ipinfo['_id'], bgppref_rec['_id'])) - else: - # ASN->Org - if 'org' in asn_rec: - org_rec = clean_secret_data(mongo.db.org.find_one({'_id': asn_rec['org']})) - if org_rec is None: - print( - "ERROR: Can't find Org '{}' in database (trying to enrich IP {}, bgppref {}, ASN {})".format( - asn_rec['org'], ipinfo['_id'], bgppref_rec['_id'], asn)) - else: - conv_dates(org_rec) - asn_rec['org'] = org_rec - - del asn_rec['bgppref'] - conv_dates(asn_rec) - asn_list.append(asn_rec) - - del bgppref_rec['asn'] - conv_dates(bgppref_rec) - ipinfo['bgppref'] = bgppref_rec - ipinfo['asn'] = asn_list - - # IP->ipblock - if 'ipblock' in ipinfo: - ipblock_rec = clean_secret_data(mongo.db.ipblock.find_one({'_id': ipinfo['ipblock']})) - if ipblock_rec is None: - print("ERROR: Can't find IP block '{}' in database (trying to enrich IP {})".format(ipinfo['ipblock'], - ipinfo['_id'])) - else: - # ipblock->org - if "org" in ipblock_rec: - org_rec = clean_secret_data(mongo.db.org.find_one({'_id': ipblock_rec['org']})) - if org_rec is None: - print("ERROR: Can't find Org '{}' in database (trying to enrich IP {}, ipblock '{}')".format( - ipblock_rec['org'], ipinfo['_id'], ipblock_rec['_id'])) - else: - conv_dates(org_rec) - ipblock_rec['org'] = org_rec - - conv_dates(ipblock_rec) - ipinfo['ipblock'] = ipblock_rec - - -def clean_secret_data(data): - """Remove all keys starting with '_' (except '_id') from dict.""" - if data is not None: - for i in list(data): - if i.startswith("_") and i != "_id": - del data[i] - return data +# def get_ip_info(ipaddr, full): +# data = { +# 'err_n': 400, +# 'error': "No IP address specified", +# 'ip': ipaddr +# } + +# if not ipaddr: +# log_err.log('400_bad_request') +# return False, Response(json.dumps(data), 400, mimetype='application/json') + +# form = SingleIPForm(ip=ipaddr) +# if not form.validate(): +# log_err.log('400_bad_request') +# data['error'] = "Bad IP address" +# return False, Response(json.dumps(data), 400, mimetype='application/json') + +# ipint = ipstr2int(form.ip.data) # Convert string IP to int + +# if full: +# ipinfo = mongo.db.ip.find_one({'_id': ipint}) +# else: +# ipinfo = mongo.db.ip.find_one({'_id': ipint}, +# {'rep': 1, 'fmp': 1, 'hostname': 1, 'bgppref': 1, 'ipblock': 1, 'geo': 1, 'bl': 1, +# 'tags': 1}) +# if not ipinfo: +# log_err.log('404_api_ip_not_found') +# data['err_n'] = 404 +# data['error'] = "IP address not found" +# return False, Response(json.dumps(data), 404, mimetype='application/json') + +# ipinfo['_id'] = int2ipstr(ipinfo['_id']) # Convert int IP to string + +# attach_whois_data(ipinfo, full) +# return True, ipinfo + +# def conv_dates(rec): +# """Convert datetimes in a record to YYYY-MM-DDTMM:HH:SS string""" +# for key in ('ts_added', 'ts_last_update'): +# if key in rec and isinstance(rec[key], datetime): +# rec[key] = rec[key].strftime("%Y-%m-%dT%H:%M:%S") + +# def clean_secret_data(data): +# """Remove all keys starting with '_' (except '_id') from dict.""" +# if data is not None: +# for i in list(data): +# if i.startswith("_") and i != "_id": +# del data[i] +# return data # ***** NERD API BasicInfo - helper funcs ***** -def get_basic_info_dic(val): - geo_d = {} - if 'geo' in val.keys(): - geo_d['ctry'] = val['geo'].get('ctry', "unknown") - - bl_l = [] - for l in val.get('bl', []): - bl_l.append(l['n']) # TODO: shouldn't there be a check for v=1? - - tags_l = [] - for l in val.get('tags', []): - d = { - 'n': l, - 'c': val['tags'][l]['confidence'] - } - - tags_l.append(d) - - data = { - 'ip': val['_id'], - 'rep': val.get('rep', 0.0), - 'fmp': val.get('fmp', {'general': 0.0}), - 'hostname': (val.get('hostname', '') or '')[::-1], - 'ipblock': val.get('ipblock', ''), - 'bgppref': val.get('bgppref', ''), - 'asn': val.get('asn', []), - 'geo': geo_d, - 'bl': bl_l, - 'tags': tags_l - } - - return data - - -def get_basic_info_dic_short(val): - # only 'rep' and 'tags' fields - tags_l = [] - for l in val.get('tags', []): - d = { - 'n': l, - 'c': val['tags'][l]['confidence'] - } - tags_l.append(d) - - data = { - 'ip': val['_id'], - 'rep': val.get('rep', 0.0), - 'tags': tags_l - } - return data +# def get_basic_info_dic(val): +# geo_d = {} +# if 'geo' in val.keys(): +# geo_d['ctry'] = val['geo'].get('ctry', "unknown") + +# bl_l = [] +# for l in val.get('bl', []): +# bl_l.append(l['n']) # TODO: shouldn't there be a check for v=1? + +# tags_l = [] +# for l in val.get('tags', []): +# d = { +# 'n': l, +# 'c': val['tags'][l]['confidence'] +# } + +# tags_l.append(d) + +# data = { +# 'ip': val['_id'], +# 'rep': val.get('rep', 0.0), +# 'fmp': val.get('fmp', {'general': 0.0}), +# 'hostname': (val.get('hostname', '') or '')[::-1], +# 'ipblock': val.get('ipblock', ''), +# 'bgppref': val.get('bgppref', ''), +# 'asn': val.get('asn', []), +# 'geo': geo_d, +# 'bl': bl_l, +# 'tags': tags_l +# } + +# return data + + +# def get_basic_info_dic_short(val): +# # only 'rep' and 'tags' fields +# tags_l = [] +# for l in val.get('tags', []): +# d = { +# 'n': l, +# 'c': val['tags'][l]['confidence'] +# } +# tags_l.append(d) + +# data = { +# 'ip': val['_id'], +# 'rep': val.get('rep', 0.0), +# 'tags': tags_l +# } +# return data # ***** NERD API BasicInfo ***** @@ -2319,6 +2248,256 @@ def get_shodan_response(ipaddr=None): # ********** +# ***************************** API v2 helper functions ******************************************** +# def create_query_v2(data): +# # Prepare 'find' part of the query +# queries = [] +# if data is None: +# return None + +# if "subnet" in data and data["subnet"] is not None and len(data["subnet"]) != 0: +# subqueries = [] +# for subnet in data["subnet"]: +# subnet = ipaddress.IPv4Network(subnet, strict=False) +# subnet_start = int(subnet.network_address) # IP addresses are stored as int +# subnet_end = int(subnet.broadcast_address) +# subqueries.append( {'$and': [{'_id': {'$gte': subnet_start}}, {'_id': {'$lte': subnet_end}}]} ) +# queries.append({'$or': subqueries}) + +# if "hostname" in data and data["hostname"] is not None and len(data["hostname"]) != 0: +# subqueries = [] +# for hostname in data["hostname"]: +# hn = hostname[::-1] # Hostnames are stored reversed in DB to allow search by suffix as a range search +# hn_end = hn[:-1] + chr(ord(hn[-1])+1) +# subqueries.append( {'$and': [{'hostname': {'$gte': hn}}, {'hostname': {'$lt': hn_end}}]} ) +# queries.append({'$or': subqueries}) + +# if "country" in data and data["country"] is not None and len(data["country"]) != 0: +# queries.append( { '$or': [{'geo.ctry': c.upper() } for c in data["country"]]} ) + +# if "asn" in data and data["asn"] is not None and len(data["asn"]) != 0 : +# subqueries = [] +# for asn in data["asn"]: +# # ASN is not stored in IP records - get list of BGP prefixes of the ASN and filter by these +# asn = int(asn.lstrip("ASas")) +# asrec = mongo.db.asn.find_one({'_id': asn}) +# if asrec and 'bgppref' in asrec: +# subqueries.append( {'bgppref': {'$in': asrec['bgppref']}} ) +# else: +# subqueries.append( {'_id': {'$exists': False}} ) # ASN not in DB, add query which is always false to get no results +# op = '$and' if (data["asn_op"] == "AND") else '$or' +# queries.append({op: subqueries}) + +# if "source" in data and data["source"] is not None and len(data["source"]) != 0: +# op = '$and' if (data["source_op"] == "AND") else '$or' +# queries.append( {op: [{'_ttl.' + s.lower(): {'$exists': True}} for s in data["source"]]} ) + +# if "cat" in data and data["cat"] is not None and len(data["cat"]) != 0: +# op = '$and' if (data["cat_op"] == "AND") else '$or' +# queries.append( {op: [{'events.cat': cat} for cat in data["cat"]]} ) + +# if "node" in data and data["node"] is not None and len(data["node"]) != 0: +# op = '$and' if (data["node_op"] == "AND") else '$or' +# queries.append( {op: [{'events.node': node} for node in data["node"]]} ) + +# if "blacklist" in data and data["blacklist"] is not None and len(data["blacklist"]) != 0: +# op = '$and' if (data["bl_op"] == "AND") else '$or' +# array = [{('dbl' if t == 'd' else 'bl'): {'$elemMatch': {'n': id, 'v': 1}}} for t,_,id in map(lambda s: s.partition(':'), data["blacklist"])] +# queries.append( {op: array} ) + +# if "tag" in data and data["tag"] is not None and len(data["tag"]) != 0: +# op = '$and' if (data["tag_op"] == "AND") else '$or' +# queries.append( {op: [{'tags.'+ tag_id: {'$exists': True}} for tag_id in data["tag"]]} ) + +# if "whitelisted" in data and data["whitelisted"]: +# queries.append( {'tags.whitelist': {'$exists': False}} ) + +# query = {'$and': queries} if queries else None +# return query + +# def get_basic_info_dic_v2(val): +# geo_d = {} +# if 'geo' in val.keys(): +# geo_d['ctry'] = val['geo'].get('ctry', "unknown") + +# bl_l = [] +# for l in val.get('bl', []): +# bl_l.append(l['n']) # TODO: shouldn't there be a check for v=1? + +# tags_l = [] +# for l in val.get('tags', []): +# d = { +# 'n' : l, +# 'c' : val['tags'][l]['confidence'] +# } + +# tags_l.append(d) + +# data = { +# 'ip' : val['_id'], +# 'rep' : val.get('rep', 0.0), +# 'fmp' : val.get('fmp', {'general': 0.0}), +# 'hostname' : (val.get('hostname', '') or '')[::-1], +# 'ipblock' : val.get('ipblock', ''), +# 'bgppref' : val.get('bgppref', ''), +# 'asn' : val.get('asn',[]), +# 'geo' : geo_d, +# 'bl' : bl_l, +# 'tags' : tags_l, +# 'ts_last_update' : val.get('ts_last_update', ''), +# 'ts_added' : val.get('ts_added', ''), +# } + +# return data + +# def find_ip_data(query, skip_n, limit): +# return mongo.db.ip.find(query).skip(skip_n).limit(limit) + +# def ip_to_warden_data(ipaddr): +# ipint = ipstr2int(ipaddr) +# ipinfo = mongo.db.ip.aggregate(pipeline = [ +# { '$match': { '_id': ipint } }, +# { '$unwind': '$events' }, +# { '$group': { +# '_id': { +# 'date': '$events.date', +# 'cat': '$events.cat' +# }, +# 'n_sum': { '$sum': '$events.n' }, +# 'conns_sum': { '$sum': '$events.conns' }, +# } +# }, +# { '$group': { +# '_id': '$_id.date', +# 'categories': { +# "$push": { +# "k": "$_id.cat", +# "v": { +# "n_sum": "$n_sum", +# "conns_sum": "$conns_sum", +# "nodes": "$nodes" +# } +# } +# } +# } +# }, +# { '$project': { +# '_id': 0, +# 'date': '$_id', +# "categories": {"$arrayToObject": "$categories"} +# } +# }, +# { '$sort': { 'date': 1 } } +# ]) + +# ipinfo_list = [doc for doc in ipinfo] +# #ipinfo_list['_id'] = int2ipstr(ipinfo_list['_id']) +# return ipinfo_list + + +# additional data for selected users that belong to group +def ip_to_warden_data_with_nodes(ipaddr): + ipint = ipstr2int(ipaddr) + ipinfo = mongo.db.ip.aggregate(pipeline = [ + { '$match': { '_id': 1246899993 } }, + { '$unwind': '$events' }, + { '$group': { + '_id': { + 'date': '$events.date', + 'cat': '$events.cat', + 'node': '$events.node' + }, + 'n_sum': { '$sum': '$events.n' }, + 'conns_sum': { '$sum': '$events.conns' } + }}, + { '$group': { + '_id': { + 'date': '$_id.date', + 'cat': '$_id.cat' + }, + 'n_sum': { '$sum': '$n_sum' }, + 'conns_sum': { '$sum': '$conns_sum' }, + 'nodes': { + '$push': { + 'node': '$_id.node', + 'n_sum': '$n_sum', + 'conns_sum': '$conns_sum' + } + } + }}, + { '$group': { + '_id': '$_id.date', + 'categories': { + '$push': { + 'cat': '$_id.cat', + 'n_sum': '$n_sum', + 'conns_sum': '$conns_sum', + 'nodes': '$nodes' + } + } + }}, + { '$project': { + '_id': 0, + 'date': '$_id', + 'categories': 1 + }}, + { '$sort': { 'date': -1 } } + ]) + + ipinfo_list = [doc for doc in ipinfo] + #ipinfo_list['_id'] = int2ipstr(ipinfo_list['_id']) + return ipinfo_list + + +# ***************************************************************************************** + +swagger_config = { + "headers": [ + ], + "specs": [ + { + "endpoint": 'apispec_1', + "route": '/apispec_1.json', + "rule_filter": lambda rule: True, # all in + "model_filter": lambda tag: True, # all in + } + ], + "static_url_path": "/flasgger_static", + # "static_folder": "static", # must be set by user + "swagger_ui": True, + "specs_route": "/apidocs/" +} +template = { + "swagger": "2.0", + "info": { + "title": "NERD API v2", + "description": "Second API version for new NERD system, built to interact with Vue.js FE", + "contact": { + "responsibleOrganization": "Liberouter", + "responsibleDeveloper": "NERD", + "email": "bartos@cesnet.cz", + "url": "https://nerd.cesnet.cz/", + }, + }, + "host": "127.0.0.1", # overrides localhost:500 + "basePath": "/doc", # base bash for blueprint registration + "schemes": [ + "http", + "https" + ], + "operationId": "getmyData" +} + +app.config['SWAGGER_VALIDATOR_ENABLE'] = True +swagger = Swagger(app, config=swagger_config, template=template, parse=True) + +# register blueprints - generally definitions of routes in separate files +from user_management import user_management +from api_v2 import api_v2 + +app.register_blueprint(user_management, url_prefix="/user") +app.register_blueprint(api_v2, url_prefix="/api/v2") + if __name__ == "__main__": # Set global testing flag config.testing = True diff --git a/NERDweb/user_management.py b/NERDweb/user_management.py new file mode 100644 index 00000000..c14f5843 --- /dev/null +++ b/NERDweb/user_management.py @@ -0,0 +1,146 @@ +from datetime import datetime, timedelta + +from flask import Blueprint, flash, redirect, session, g, make_response, current_app, url_for, request, jsonify +from flask_wtf import FlaskForm +from wtforms import StringField, PasswordField, SubmitField +from wtforms.validators import DataRequired, Email, EqualTo, Length, Regexp +from bcrypt import hashpw, checkpw, gensalt +from flask_mail import Message +from itsdangerous import URLSafeTimedSerializer, BadSignature, SignatureExpired +from flask_dance.consumer import OAuth2ConsumerBlueprint +import jwt +import os +import sys +import common.config + + +# needed overridden render_template method because it passes some needed attributes to Jinja templates +from com import mailer +from userdb import create_user, get_user_data_for_login, get_user_by_email, verify_user, get_verification_email_sent, \ + set_verification_email_sent, set_last_login, get_user_name, set_new_password, check_if_user_exists, get_user_by_id + +ERROR_MSG_MISSING_MAIL_CONFIG = "ERROR: No destination email address configured. This is a server configuration " \ + "error. Please, report this to NERD administrator if possible." + +# variable name and name of blueprint is recommended to be same as filename +user_management = Blueprint("user_management", __name__, static_folder="static", template_folder="templates") + + +DEFAULT_CONFIG_FILE = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "/etc/nerd/nerdweb.yml")) + +if len(sys.argv) >= 2: + cfg_file = sys.argv[1] +else: + cfg_file = DEFAULT_CONFIG_FILE +cfg_dir = os.path.dirname(os.path.abspath(cfg_file)) + +# Read web-specific config (nerdweb.cfg) +config = common.config.read_config(cfg_file) +# Read common config (nerd.cfg) and combine them together +common_cfg_file = os.path.join(cfg_dir, config.get('common_config')) +config.update(common.config.read_config(common_cfg_file)) + +config.testing = True + +# ***** Util functions ***** +def generate_token(user_email): + """ Generates random token for email verification or password reset. """ + serializer = URLSafeTimedSerializer(current_app.config['SECRET_KEY']) + return serializer.dumps(user_email) + +def generate_jwt_token(email): + out = {} + out['user'] = { + 'login_type': 'local', + 'id': 'local:' + email, + 'email': email, + } + out['exp'] = datetime.utcnow()+timedelta(hours=4) + return jwt.encode(out, config.get('secret_key'), algorithm="HS256") + +def confirm_jwt_token(token): + try: + data=jwt.decode(token, config.get("secret_key"), algorithms=["HS256"]) + + if data["user"]["login_type"] == "devel": + return data + current_user=get_user_by_id(data["user"]["id"]) + if current_user is None: + return { + "message": "Invalid Authentication token!", + "data": data["user"]["id"], + "error": "Unauthorized" + }, 401 + if not "registered" in current_user["groups"]: + return { + "message": "Email address not verified", + "data": None, + }, 403 + except Exception as e: + return { + "message": "Something went wrong", + "data": str(e), + "error": "AuthToken wrong" + }, 401 + return current_user + + +def confirm_token(token, expiration=3600): + serializer = URLSafeTimedSerializer(current_app.config['SECRET_KEY']) + try: + email = serializer.loads( + token, + max_age=expiration + ) + except (BadSignature, SignatureExpired): + return False + return email + + +def get_current_datetime(): + return datetime.now() # TODO? shouldn't it be utcnow()? uklad�d� se do DB + + +def send_verification_email(user_email, name): + # TODO: udelat nekde kontrolu, ze je nakonfigurovane mailovani + token = generate_token(user_email) + confirm_url = url_for('user_management.verify_email', token=token, _external=True) + msg = Message(subject="[NERD] Verify your account email address", + recipients=[user_email], + reply_to=current_app.config.get('MAIL_DEFAULT_SENDER'), + body=f"Dear {name},\n\nplease verify your email address to complete your registration process by" + f" clicking on this link:\n\n{confirm_url}\n\nThank you,\nNERD administrator", + ) + mailer.send(msg) + + +def send_password_reset_email(user_email, name): + if not config.get('login.request-email', None): + return make_response(ERROR_MSG_MISSING_MAIL_CONFIG) + token = generate_token(user_email) + password_reset_url = url_for('user_management.password_reset', token=token, _external=True) + msg = Message(subject="[NERD] Password reset request", + recipients=[user_email], + reply_to=current_app.config.get('MAIL_DEFAULT_SENDER'), + body=f"Dear {name},\n\nyou can reset your password by clicking on this link:" + f"\n\n{password_reset_url}\n\nThank you,\nNERD administrator", + ) + mailer.send(msg) + + +def get_hashed_password(password): + return hashpw(password.encode('utf-8'), gensalt()).decode('utf-8') + + +def verify_email_token(token): + """ Verifies email token and returns user's email, to which the token was crafted and also user info. """ + email = confirm_token(token) + if not email: + flash('The link is invalid or has expired.', 'error') + return None + user = get_user_by_email(email) + if user is None: + flash('Such user account does not exist. Please check your link and if the problem persists, ' + 'contact NERD administrator.', 'error') + return user + diff --git a/NERDweb/userdb.py b/NERDweb/userdb.py index 1af598d2..0fd0fca3 100644 --- a/NERDweb/userdb.py +++ b/NERDweb/userdb.py @@ -51,7 +51,6 @@ def get_all_groups(): groups.discard('*') return sorted(list(groups)) - # ***** Access control functions ***** def get_user_groups(full_id): @@ -77,9 +76,6 @@ def ac(resource): return False return ac - -# TODO - split authentication and authorization/get_user_information - def get_user_info(session): """ Returun info about current user (or None if noone is logged in) and @@ -167,3 +163,155 @@ def generate_unique_token(user): return False return True + +############################################### +# NEW FUNCTIONS API v2 # +############################################### +# ***** User management functions ***** +def create_user(email, password, provider, name=None, organization=None, groups=[]): + try: + cur = db.cursor() + cur.execute("""INSERT INTO users (id, groups, name, email, org, password) + VALUES (%s, %s, %s, %s, %s, %s)""", + (provider + ":" + email, groups, name, email, organization, password)) + db.commit() + cur.close() + except (Exception, psycopg2.DatabaseError) as e: + return e + +def get_user_by_email(email): + cur = db.cursor() + cur.execute("SELECT * FROM users WHERE email=%s", (email,)) + row = cur.fetchone() + if not row: + return None + col_names = [col.name for col in cur.description] + return dict(zip(col_names, row)) + +def get_user_by_id(id): + cur = db.cursor() + cur.execute("SELECT * FROM users WHERE id=%s", (id,)) + row = cur.fetchone() + if not row: + return None + col_names = [col.name for col in cur.description] + return dict(zip(col_names, row)) + +def get_user_data_for_login(user_id): + cur = db.cursor() + cur.execute("SELECT id, password, name FROM users WHERE id = %s", (user_id,)) + row = cur.fetchone() + if not row: + return None + return {'id': row[0], 'password': row[1], 'name': row[2]} + +def check_if_user_exists(user_id): + cur = db.cursor() + cur.execute("SELECT id, name FROM users WHERE id = %s", (user_id,)) + row = cur.fetchone() + if not row: + return False + return True + +def verify_user(user_id): + try: + cur = db.cursor() + cur.execute("""UPDATE users SET groups=%s, verified=TRUE WHERE id = %s""", (["registered"], user_id,)) + db.commit() + cur.close() + except psycopg2.Error as e: + print(f"verify_user() failed: {e.pgerror}") + return e + +def verify_user_by_mail(mail): + try: + cur = db.cursor() + cur.execute("""UPDATE users SET groups=%s, verified=TRUE WHERE email = %s""", (["registered"], mail,)) + db.commit() + cur.close() + except psycopg2.Error as e: + print(f"verify_user() failed: {e.pgerror}") + return e + +def just_verify_user_by_id(ide): + try: + cur = db.cursor() + cur.execute("""UPDATE users SET verified=TRUE WHERE id = %s""", (ide,)) + db.commit() + cur.close() + except psycopg2.Error as e: + print(f"verify_user() failed: {e.pgerror}") + return e + +def set_verification_email_sent(date_time, email): + try: + cur = db.cursor() + cur.execute("""UPDATE users SET verification_email_sent=%s WHERE id = %s""", (date_time, id)) + db.commit() + cur.close() + except psycopg2.Error as e: + print(f"set_verification_email_sent() failed: {e.pgerror}") + return e + + +def set_last_login(date_time, ide): + try: + cur = db.cursor() + cur.execute("""UPDATE users SET last_login=%s WHERE id = %s""", (date_time, ide)) + db.commit() + cur.close() + except psycopg2.Error as e: + print(f"set_last_login() failed: {e.pgerror}") + return e + + +def get_verification_email_sent(user_email): + cur = db.cursor() + cur.execute("SELECT verification_email_sent FROM users WHERE email = %s", (user_email,)) + row = cur.fetchone() + if not row: + return None + return row[0] + + +def get_user_name(user_email): + cur = db.cursor() + cur.execute("SELECT name FROM users WHERE email = %s", (user_email,)) + row = cur.fetchone() + if not row: + return None + return row[0] + +def set_new_password(new_password, id): + try: + cur = db.cursor() + cur.execute("""UPDATE users SET password=%s WHERE id = %s""", (new_password, id)) + db.commit() + cur.close() + except psycopg2.Error as e: + print(f"set_new_password() failed: {e.pgerror}") + return e + + +def set_api_v1_token(ide, token): + cur = db.cursor() + cur.execute("UPDATE users SET api_token = %s WHERE id = %s", (token, ide,)) + return True + +def get_users_admin(): + cur = db.cursor() + cur.execute("SELECT email, groups, id, org, api_token, verified, verification_email_sent, last_login FROM users ORDER BY email") + row = cur.fetchall() + if not row: + return None + return row + +def set_new_roles(ide, roles): + cur = db.cursor() + cur.execute("UPDATE users SET groups = %s WHERE id = %s", (roles, ide,)) + return True + +def delete_user(ide): + cur = db.cursor() + cur.execute("DELETE FROM users WHERE id = %s", (ide,)) + return True \ No newline at end of file