panariga/xbotcontrol
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added timestampOverride for manual log process

Browse Source
This commit is contained in:
panariga
2024-12-18 10:26:40 +02:00
parent 141deaa35b
commit b8446181d3
5 changed files with 352 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
smarty/compile/affb24851ed623b62affa076808377b28b01c478_2.file_index.tpl.php
View File

@@ -1,18 +1,18 @@
<?php
/* Smarty version 5.4.2, created on 2024-12-17 11:29:28
/* Smarty version 5.4.2, created on 2024-12-17 20:54:57
from 'file:index.tpl' */
/* @var \Smarty\Template $_smarty_tpl */
if ($_smarty_tpl->getCompiled()->isFresh($_smarty_tpl, array (
'version' => '5.4.2',
'unifunc' => 'content_676160989e3635_07407148',
'unifunc' => 'content_6761e5219d7cf3_55254235',
'has_nocache_code' => false,
'file_dependency' =>
array (
'affb24851ed623b62affa076808377b28b01c478' =>
array (
0 => 'index.tpl',
1 => 1734434957,
1 => 1734468879,
2 => 'file',
),
),
@@ -20,7 +20,7 @@ if ($_smarty_tpl->getCompiled()->isFresh($_smarty_tpl, array (
array (
),
))) {
function content_676160989e3635_07407148 (\Smarty\Template $_smarty_tpl) {
function content_6761e5219d7cf3_55254235 (\Smarty\Template $_smarty_tpl) {
$_smarty_current_dir = '/home/l/public_html/xbotcontrol/smarty/template';
?><!DOCTYPE html>
@@ -76,43 +76,26 @@ $_smarty_current_dir = '/home/l/public_html/xbotcontrol/smarty/template';
<a class="btn text-light" onclick="initializeTable('latest_requests');">Latest</a>
</li>
<li class="nav-item">
<a class="btn text-light" href="<?php echo htmlspecialchars((string) ($_ENV['BASEURI']), ENT_QUOTES, 'UTF-8');?>
/contacts"><?php echo htmlspecialchars((string) ($_smarty_tpl->getConfigVariable('linkContacts')), ENT_QUOTES, 'UTF-8');?>
</a>
<a class="btn text-light" onclick="initializeTable('count_requests_by_ip');">Top by IP</a>
</li>
<li class="nav-item">
<a class="btn text-light" href="<?php echo htmlspecialchars((string) ($_ENV['BASEURI']), ENT_QUOTES, 'UTF-8');?>
/lists/read"><?php echo htmlspecialchars((string) ($_smarty_tpl->getConfigVariable('linkLists')), ENT_QUOTES, 'UTF-8');?>
</a>
<a class="btn text-light" onclick="initializeTable('count_requests_by_ua');">Top by UA</a>
</li>
<?php if ($_SESSION['user_role'] == 'admin') {?>
<li class="nav-item">
<a class="btn text-light"
href="<?php echo htmlspecialchars((string) ($_ENV['BASEURI']), ENT_QUOTES, 'UTF-8');?>
/lists/show_membership"><?php echo htmlspecialchars((string) ($_smarty_tpl->getConfigVariable('linkMembership')), ENT_QUOTES, 'UTF-8');?>
</a>
<a class="btn text-light" onclick="initializeTable('top_ip_ua_path');">IP+UA+Path</a>
</li>
<li class="nav-item">
<a class="btn text-light" href="<?php echo htmlspecialchars((string) ($_ENV['BASEURI']), ENT_QUOTES, 'UTF-8');?>
/users/read"><?php echo htmlspecialchars((string) ($_smarty_tpl->getConfigVariable('linkUsers')), ENT_QUOTES, 'UTF-8');?>
</a>
<a class="btn text-light" onclick="initializeTable('top_ip_by_load');">IP+Load</a>
</li>
<li class="nav-item">
<a class="btn text-light" href="<?php echo htmlspecialchars((string) ($_ENV['BASEURI']), ENT_QUOTES, 'UTF-8');?>
/log/read"><?php echo htmlspecialchars((string) ($_smarty_tpl->getConfigVariable('linkLog')), ENT_QUOTES, 'UTF-8');?>
</a>
<a class="btn text-light" onclick="initializeTable('top_ip_by_rps');">IP+RPS</a>
</li>
<li class="nav-item">
<a class="btn text-light" onclick="initializeTable('top_net_28_by_rps');">IP+RPS</a>
</li>
<?php }?>
</ul>
<a class="btn btn-outline-success" href="<?php echo htmlspecialchars((string) ($_ENV['BASEURI']), ENT_QUOTES, 'UTF-8');?>
/login?logout=true"><?php echo htmlspecialchars((string) ($_smarty_tpl->getConfigVariable('logOutbtn')), ENT_QUOTES, 'UTF-8');?>
(<?php echo htmlspecialchars((string) ($_SESSION['username']), ENT_QUOTES, 'UTF-8');?>
)</a>
</div>
</div>

22
smarty/template/index.tpl
View File

@@ -42,30 +42,26 @@
<a class="btn text-light" onclick="initializeTable('latest_requests');">Latest</a>
</li>
<li class="nav-item">
<a class="btn text-light" href="{$smarty.env.BASEURI}/contacts">{#linkContacts#}</a>
<a class="btn text-light" onclick="initializeTable('count_requests_by_ip');">Top by IP</a>
</li>
<li class="nav-item">
<a class="btn text-light" href="{$smarty.env.BASEURI}/lists/read">{#linkLists#}</a>
<a class="btn text-light" onclick="initializeTable('count_requests_by_ua');">Top by UA</a>
</li>
{if $smarty.session.user_role == 'admin'}
<li class="nav-item">
<a class="btn text-light"
href="{$smarty.env.BASEURI}/lists/show_membership">{#linkMembership#}</a>
<a class="btn text-light" onclick="initializeTable('top_ip_ua_path');">IP+UA+Path</a>
</li>
<li class="nav-item">
<a class="btn text-light" href="{$smarty.env.BASEURI}/users/read">{#linkUsers#}</a>
<a class="btn text-light" onclick="initializeTable('top_ip_by_load');">IP+Load</a>
</li>
<li class="nav-item">
<a class="btn text-light" href="{$smarty.env.BASEURI}/log/read">{#linkLog#}</a>
<a class="btn text-light" onclick="initializeTable('top_ip_by_rps');">IP+RPS</a>
</li>
<li class="nav-item">
<a class="btn text-light" onclick="initializeTable('top_net_28_by_rps');">IP+RPS</a>
</li>
{/if}
</ul>
<a class="btn btn-outline-success" href="{$smarty.env.BASEURI}/login?logout=true">{#logOutbtn#}
({$smarty.session.username})</a>
</div>
</div>

413
src/Classes/Report.php
View File

@@ -4,124 +4,29 @@ declare(strict_types=1);
namespace XBotControl\Classes;
use React\Promise\PromiseInterface;
use Psr\Http\Message\ServerRequestInterface;
class Report
{
public static function latest_requests(ServerRequestInterface $request): PromiseInterface
private static function generateColumns(array $definitions): array
{
$columnsDefinition = [
[
'title' => 'id',
'field' => 'id',
'visible' => false,
'sortable' => true,
'filterControl' => 'input',
'widthUnit' => 'input',
'width' => 'input',
],
$columns = [];
foreach ($definitions as $definition) {
$columns[] = array_merge(
[
'sortable' => true,
'title' => 'ip',
'field' => 'ip',
'sortable' => true,
'visible' => true,
'filterControl' => 'input',
],
[
'sortable' => true,
'title' => 'domain',
'field' => 'domain',
'sortable' => true,
'visible' => false,
'filterControl' => 'input',
],
[
'sortable' => true,
'title' => 'path',
'field' => 'path',
'sortable' => true,
'filterControl' => 'input',
],
[
'sortable' => true,
'title' => 'useragent',
'field' => 'useragent',
'sortable' => true,
'filterControl' => 'input',
],
[
'sortable' => true,
'title' => 'load',
'field' => 'load',
'sortable' => true,
'filterControl' => 'input',
],
[
'sortable' => true,
'title' => 'datetime',
'field' => 'datetime',
'sortable' => true,
'filterControl' => 'input',
],
];
$sql = "SELECT
req.rowid AS id,
ip.data AS ip,
domain.data AS domain,
path.data AS path,
useragent.data AS useragent,
headers.data AS headers ,
(SELECT load.load1
FROM load
WHERE load.rowid >= req.timestamp
ORDER BY load.rowid DESC LIMIT 1) AS load,
datetime(req.timestamp, 'auto') AS datetime
FROM
request req
LEFT JOIN
ip ON req.id_ip = ip.rowid
LEFT JOIN
domain ON req.id_domain = domain.rowid
LEFT JOIN
path ON req.id_path = path.rowid
LEFT JOIN
useragent ON req.id_useragent = useragent.rowid
LEFT JOIN
headers ON req.id_headers = headers.rowid
WHERE 1=1 ";
$params = [];
$query = $request->getQueryParams();
if (isset($query['filter'])) {
$filter = json_decode($request->getQueryParams()['filter'], true);
} else {
$filter = [];
$definition
);
}
return $columns;
}
foreach ($filter as $field => $value) {
$sql .= 'AND ' . $field . ' LIKE ? ';
$params[] = '%' . $value . '%';
}
$sql .= " AND req.timestamp BETWEEN ? AND ? ";
$sql .= ' ORDER BY req.rowid DESC ';
$sql .= ' LIMIT ? ;';
$params[] = strtotime($request->getQueryParams()['from'] ?? 'yesterday');
$params[] = strtotime($request->getQueryParams()['to'] ?? 'now');
$params[] = (int)$request->getQueryParams()['limit'] ?? 100;
private static function executeQuery(string $sql, array $params, array $columnsDefinition): PromiseInterface
{
return \XBotControl\Storage::getInstance()->db->query($sql, $params)->then(function ($result) use ($columnsDefinition) {
return [
"columns" => $columnsDefinition,
@@ -129,4 +34,300 @@ class Report
];
});
}
private static function parseQueryParams(ServerRequestInterface $request): array
{
$query = $request->getQueryParams();
return [
'from' => strtotime($query['from'] ?? 'yesterday'),
'to' => strtotime($query['to'] ?? 'now'),
'limit' => (int)($query['limit'] ?? 100),
'filter' => isset($query['filter']) ? json_decode($query['filter'], true) : []
];
}
private static function prepareFilterClauses(array $filter): array
{
$sql = '';
$params = [];
foreach ($filter as $field => $value) {
$sql .= 'AND ' . $field . ' LIKE ? ';
$params[] = '%' . $value . '%';
}
return [$sql, $params];
}
public static function latest_requests(ServerRequestInterface $request): PromiseInterface
{
$columnsDefinition = self::generateColumns([
["title" => "id", "field" => "id", "visible" => false],
["title" => "ip", "field" => "ip"],
["title" => "domain", "field" => "domain", "visible" => false],
["title" => "path", "field" => "path"],
["title" => "useragent", "field" => "useragent"],
["title" => "load", "field" => "load"],
["title" => "datetime", "field" => "datetime"],
]);
$queryParams = self::parseQueryParams($request);
$sql = "
SELECT
req.rowid AS id, ip.data AS ip, domain.data AS domain,
path.data AS path, useragent.data AS useragent,
headers.data AS headers,
(SELECT load.load1 FROM load WHERE load.rowid >= req.timestamp ORDER BY load.rowid DESC LIMIT 1) AS load,
datetime(req.timestamp, 'auto') AS datetime
FROM
request req
LEFT JOIN ip ON req.id_ip = ip.rowid
LEFT JOIN domain ON req.id_domain = domain.rowid
LEFT JOIN path ON req.id_path = path.rowid
LEFT JOIN useragent ON req.id_useragent = useragent.rowid
LEFT JOIN headers ON req.id_headers = headers.rowid
WHERE 1=1
";
list($filterSQL, $filterParams) = self::prepareFilterClauses($queryParams['filter']);
$sql .= $filterSQL . " AND req.timestamp BETWEEN ? AND ? ORDER BY req.rowid DESC LIMIT ?;";
$params = array_merge($filterParams, [$queryParams['from'], $queryParams['to'], $queryParams['limit']]);
return self::executeQuery($sql, $params, $columnsDefinition);
}
public static function count_requests_by_ip(ServerRequestInterface $request): PromiseInterface
{
$columnsDefinition = self::generateColumns([
["title" => "ip", "field" => "ip_address"],
["title" => "request_count", "field" => "request_count"],
]);
$queryParams = self::parseQueryParams($request);
$sql = "
SELECT
ip.data AS ip_address,
COUNT(request.id_ip) AS request_count
FROM
request
INNER JOIN
ip ON request.id_ip = ip.rowid
WHERE
request.timestamp BETWEEN ? AND ?
GROUP BY
ip.data
ORDER BY
request_count DESC
LIMIT ?;
";
$params = [$queryParams['from'], $queryParams['to'], $queryParams['limit']];
return self::executeQuery($sql, $params, $columnsDefinition);
}
public static function count_requests_by_ua(ServerRequestInterface $request): PromiseInterface
{
$columnsDefinition = self::generateColumns([
["title" => "useragent", "field" => "id_useragent"],
["title" => "request_count", "field" => "request_count"],
]);
$queryParams = self::parseQueryParams($request);
$sql = "
SELECT
useragent.data AS id_useragent,
COUNT(request.id_useragent) AS request_count
FROM
request
INNER JOIN
useragent ON request.id_useragent = useragent.rowid
WHERE
request.timestamp BETWEEN ? AND ?
GROUP BY
useragent.data
ORDER BY
request_count DESC
LIMIT ?;
";
$params = [$queryParams['from'], $queryParams['to'], $queryParams['limit']];
return self::executeQuery($sql, $params, $columnsDefinition);
}
public static function top_ip_ua_path(ServerRequestInterface $request): PromiseInterface
{
$columnsDefinition = self::generateColumns([
["title" => "ip", "field" => "ip"],
["title" => "useragent", "field" => "user_agent"],
["title" => "path", "field" => "path"],
["title" => "count", "field" => "count"],
]);
$queryParams = self::parseQueryParams($request);
$sql = "
SELECT
ip.data AS ip,
useragent.data AS user_agent,
path.data AS path,
COUNT(request.rowid) AS count
FROM
request
JOIN ip ON request.id_ip = ip.rowid
JOIN useragent ON request.id_useragent = useragent.rowid
JOIN path ON request.id_path = path.rowid
WHERE
request.timestamp BETWEEN ? AND ?
GROUP BY
ip.data, useragent.data, path.data
ORDER BY
count DESC
LIMIT ?;
";
$params = [$queryParams['from'], $queryParams['to'], $queryParams['limit']];
return self::executeQuery($sql, $params, $columnsDefinition);
}
public static function top_ip_by_load(ServerRequestInterface $request): PromiseInterface
{
$columnsDefinition = self::generateColumns([
["title" => "ip", "field" => "data"],
["title" => "avg_load", "field" => "avg_load"],
["title" => "request_count", "field" => "request_count"],
]);
$queryParams = self::parseQueryParams($request);
$sql = "
SELECT
ip.data,
COUNT(request.rowid) AS request_count,
AVG(load.load1) AS avg_load
FROM
request
JOIN ip ON request.id_ip = ip.rowid
JOIN load ON load.rowid = (
SELECT MIN(load_sub.rowid)
FROM load AS load_sub
WHERE load_sub.rowid > request.timestamp
)
WHERE
load.load1 > 1
AND request.timestamp BETWEEN ? AND ?
GROUP BY
ip.data
ORDER BY
avg_load DESC, request_count DESC
LIMIT ?;
";
$params = [$queryParams['from'], $queryParams['to'], $queryParams['limit']];
return self::executeQuery($sql, $params, $columnsDefinition);
}
public static function top_ip_by_rps(ServerRequestInterface $request): PromiseInterface
{
$columnsDefinition = self::generateColumns([
["title" => "ip", "field" => "ip_address"],
["title" => "avg_request_per_second", "field" => "avg_request_per_second"],
]);
$queryParams = self::parseQueryParams($request);
$sql = "
WITH TimestampIPRequests AS (
SELECT
id_ip,
timestamp,
COUNT(*) AS request_count
FROM
request
WHERE
request.timestamp BETWEEN ? AND ?
GROUP BY
id_ip, timestamp
HAVING
COUNT(*) > 1
),
IPRequestPerSecond AS (
SELECT
id_ip,
AVG(request_count) AS avg_request_per_second
FROM
TimestampIPRequests
GROUP BY
id_ip
)
SELECT
ip.data as ip_address,
avg_request_per_second
FROM
IPRequestPerSecond
JOIN ip ON IPRequestPerSecond.id_ip = ip.rowid
ORDER BY
avg_request_per_second DESC
LIMIT ?;
";
$params = [$queryParams['from'], $queryParams['to'], $queryParams['limit']];
return self::executeQuery($sql, $params, $columnsDefinition);
}
public static function top_net_28_by_rps(ServerRequestInterface $request): PromiseInterface
{
$columnsDefinition = self::generateColumns([
["title" => "ip", "field" => "network"],
["title" => "avg_request_per_second", "field" => "avg_request_per_second"],
]);
$queryParams = self::parseQueryParams($request);
$sql = "
CREATE FUNCTION cidr_to_network(cidr VARCHAR(30), prefix INT) RETURNS VARCHAR(30)
BEGIN
RETURN inet_ntoa(inet_aton(substring_index(cidr, '/', 1)) & ((2 ^ (32 - prefix)) - 1 ^ 0xFFFFFFFF)) || '/' || prefix;
END;
WITH TimestampNetworkRequests AS (
SELECT
CAST(cidr_to_network(ip.data, 28) AS TEXT) AS network,
timestamp,
COUNT(*) AS request_count
FROM
request
JOIN
ip ON request.id_ip = ip.rowid
WHERE
request.timestamp BETWEEN ? AND ?
GROUP BY
network, timestamp
HAVING
COUNT(*) > 1
),
NetworkRequestPerSecond AS (
SELECT
network,
AVG(request_count) AS avg_request_per_second
FROM
TimestampNetworkRequests
GROUP BY
network
)
SELECT
network AS network_address,
avg_request_per_second
FROM
NetworkRequestPerSecond
ORDER BY
avg_request_per_second DESC
LIMIT ?;
";
$params = [$queryParams['from'], $queryParams['to'], $queryParams['limit']];
return self::executeQuery($sql, $params, $columnsDefinition);
}
}

2
src/InitTables.php
View File

@@ -23,7 +23,7 @@ class InitTables
})->then(function () use ($db) {
return $db->exec('CREATE TABLE IF NOT EXISTS headers ( data TEXT UNIQUE NOT NULL) STRICT ;');
})->then(function () use ($db) {
return $db->exec("CREATE TABLE IF NOT EXISTS networkwhitelist ( data TEXT UNIQUE NOT NULL CHECK (data LIKE '%/%'), CONSTRAINT valid_network CHECK ( data LIKE '%.%/%' OR data LIKE '%:%/%' )) STRICT ;");
return $db->exec("CREATE TABLE IF NOT EXISTS networkwhitelist ( data TEXT UNIQUE NOT NULL CHECK (data LIKE '%/%'), source TEXT NULL , CONSTRAINT valid_network CHECK ( data LIKE '%.%/%' OR data LIKE '%:%/%' )) STRICT ;");
})->then(function () use ($db) {
return $db->exec('CREATE TABLE IF NOT EXISTS request ( id_ip INTEGER NOT NULL, id_method INTEGER NOT NULL, id_domain INTEGER NOT NULL, id_path INTEGER NOT NULL, id_useragent INTEGER NOT NULL, id_headers INTEGER NOT NULL, timestamp INTEGER NOT NULL, FOREIGN KEY (id_ip) REFERENCES ip(rowid), FOREIGN KEY (id_domain) REFERENCES domain(rowid), FOREIGN KEY (id_path) REFERENCES path(rowid), FOREIGN KEY (id_useragent) REFERENCES useragent(rowid), FOREIGN KEY (id_headers) REFERENCES headers(rowid) ) STRICT ;');
})->then(function () use ($db) {

8
src/Request.php
View File

@@ -30,7 +30,7 @@ class Request
public static function save(ServerRequestInterface $request): PromiseInterface
public static function save(ServerRequestInterface $request, ?int $timestampOverride = null): PromiseInterface
{
$realIp = self::getRealIP($request);
$userAgent = $request->getHeaderLine('User-Agent') ?: 'Unknown';
@@ -45,6 +45,8 @@ class Request
'id_path' => $storage::getId('path', '/' . $request->getAttribute('original_uri', '')),
'id_useragent' => $storage::getId('useragent', $userAgent),
'id_headers' => 0,
'id_method' => self::METHOD[$request->getMethod()] ?? 0,
'timestamp' => $timestampOverride ?? time(),
];
if ($_ENV['SAVE_HEADERS'] === true) {
@@ -56,8 +58,8 @@ class Request
return \React\Promise\all($idPromises)
->then(function ($resolvedValues) use ($request, $storage) {
// Set resolved values efficiently
$resolvedValues['id_method'] = self::METHOD[$request->getMethod()] ?? 0;
$resolvedValues['timestamp'] = time();
// $resolvedValues['id_method'] = self::METHOD[$request->getMethod()] ?? 0;
// $resolvedValues['timestamp'] = time();
// Directly save data asynchronously
return $storage::insert('request', $resolvedValues);