Extracting your data

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77

import requests
import os
import json
import calendar
from datetime import datetime, timedelta, timezone

# Set up environment variables
NGWAF_EMAIL = os.getenv('NGWAF_USER_EMAIL')
NGWAF_TOKEN = os.getenv('NGWAF_TOKEN')
NGWAF_CORP = os.getenv('CORP_NAME')
NGWAF_SITE = os.getenv('SITE_NAME')

if not NGWAF_EMAIL or not NGWAF_TOKEN or not NGWAF_CORP or not NGWAF_SITE:
    raise EnvironmentError("Please set NGWAF_EMAIL, NGWAF_TOKEN, NGWAF_CORP, and NGWAF_SITE environment variables.")

# Base URL for the API
base_url = 'https://dashboard.signalsciences.net/api/v0'

# Set up headers with authentication
headers = {
    'x-api-user': NGWAF_EMAIL,
    'x-api-token': NGWAF_TOKEN
}

# Calculate UTC timestamps for the previous full hour
until_time = datetime.now(timezone.utc).replace(minute=0, second=0, microsecond=0)
from_time = until_time - timedelta(hours=1)
until_time = calendar.timegm(until_time.utctimetuple())
from_time = calendar.timegm(from_time.utctimetuple())

# Set up the initial URL for the GET request
get_url = f'{base_url}/corps/{NGWAF_CORP}/sites/{NGWAF_SITE}/feed/requests?from={from_time}&until={until_time}'

# Debugging: print the URL and timestamps
print(f"Fetching data from: {get_url}")
print(f"from_time: {from_time}, until_time: {until_time}")

def fetch_paginated_data(url):
    data_list = []
    while url:
        # Make the initial GET request
        response_raw = requests.get(url, headers=headers)
        if response_raw.status_code != 200:
            raise RuntimeError(f"Failed to fetch data from {url}. Status Code: {response_raw.status_code}")
        
        response = response_raw.json()
        data_list.extend(response.get('data', []))
        
        next_uri = response.get('next', {}).get('uri', '')
        if not next_uri:
            break

        # Extract the next parameter from the URI
        next_value = next_uri.split('next=')[-1]

        # Prepare the POST request for pagination
        post_url = f'{base_url}/corps/{NGWAF_CORP}/sites/{NGWAF_SITE}/feed/requests'
        post_data = {'next': next_value}
        headers['Content-Type'] = 'application/x-www-form-urlencoded'  # Add the necessary header
        post_response_raw = requests.post(post_url, headers=headers, data=post_data)
        if post_response_raw.status_code != 200:
            raise RuntimeError(f"Failed to fetch paginated data from {post_url}. Status Code: {post_response_raw.status_code}")
        
        post_response = post_response_raw.json()
        data_list.extend(post_response.get('data', []))
        
        next_uri = post_response.get('next', {}).get('uri', '')
        if not next_uri:
            break

    return data_list

# Fetch data
data = fetch_paginated_data(get_url)

# Output the data or save to a file, etc.
print(json.dumps(data, indent=4))