Browse Source

Caching, speedup, logging

pull/5/merge
Kai Kretschmann 4 years ago
parent
commit
dde1cbcde9
10 changed files with 376 additions and 50 deletions
  1. 1
    1
      .gitignore
  2. 24
    0
      admin/normalize.php
  3. 1
    0
      cache/.htaccess
  4. 0
    0
      cache/index.html
  5. 89
    5
      doc/db.sql
  6. 160
    43
      inc/lggr_class.php
  7. 51
    0
      inc/lggrcache_class.php
  8. 37
    0
      inc/lggrperf_class.php
  9. 3
    0
      index.php
  10. 10
    1
      tpl/foot.inc.php

+ 1
- 1
.gitignore View File

@@ -1,2 +1,2 @@
t.php
cache/key_*

+ 24
- 0
admin/normalize.php View File

@@ -0,0 +1,24 @@
<?php

spl_autoload_register(function($class) {
include __DIR__ . '/../inc/' . strtolower($class) . '_class.php';
});

$iCount=0;
$a=array();
$l = null;
try {
$config = new AdminConfig();

$state = new LggrState();
$state->setLocalCall(true);

$l = new Lggr($state, $config);

$l->normalizeHosts();

$a = $l->getPerf();
} catch(Exception $e) {
die($e->getMessage());
} // try


+ 1
- 0
cache/.htaccess View File

@@ -0,0 +1 @@
deny from all

+ 0
- 0
cache/index.html View File


+ 89
- 5
doc/db.sql View File

@@ -1,27 +1,110 @@
-- --------------------------------------------------------
-- Host: 127.0.0.1
-- Server Version: 5.5.43-0+deb7u1 - (Debian)
-- Server Betriebssystem: debian-linux-gnu
-- HeidiSQL Version: 9.2.0.4971
-- --------------------------------------------------------
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET NAMES utf8mb4 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
-- Exportiere Struktur von Tabelle logger.hosts
CREATE TABLE IF NOT EXISTS `hosts` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`name` varchar(50) NOT NULL DEFAULT '0',
PRIMARY KEY (`id`),
UNIQUE KEY `name` (`name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-- Daten Export vom Benutzer nicht ausgewählt
-- Exportiere Struktur von View logger.LastHour
-- Erstelle temporäre Tabelle um View Abhängigkeiten zuvorzukommen
CREATE TABLE `LastHour` (
`id` BIGINT(20) NOT NULL,
`date` DATETIME NOT NULL,
`facility` ENUM('kern','user','mail','daemon','auth','syslog','lpr','news','uucp','authpriv','ftp','cron','local0','local1','local2','local3','local4','local5','local6','local7') NOT NULL COLLATE 'utf8_general_ci',
`level` ENUM('emerg','alert','crit','err','warning','notice','info','debug') NOT NULL COLLATE 'utf8_general_ci',
`host` VARCHAR(50) NOT NULL COLLATE 'utf8_general_ci',
`program` VARCHAR(50) NOT NULL COLLATE 'utf8_general_ci',
`pid` INT(10) UNSIGNED NOT NULL,
`message` TEXT NOT NULL COLLATE 'utf8_general_ci',
`idhost` INT(11) NULL
) ENGINE=MyISAM;
-- Exportiere Struktur von Tabelle logger.newlogs
CREATE TABLE IF NOT EXISTS `newlogs` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`date` datetime NOT NULL,
`facility` enum('kern','user','mail','daemon','auth','syslog','lpr','news','uucp','authpriv','ftp','cron','local0','local1','local2','local3','local4','local5','local6','local7') NOT NULL,
`level` enum('emerg','alert','crit','err','warning','notice','info','debug') NOT NULL,
`host` char(16) NOT NULL,
`host` varchar(50) NOT NULL,
`program` varchar(50) NOT NULL,
`pid` int(10) unsigned NOT NULL,
`message` text NOT NULL,
`idhost` int(11) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `level` (`level`),
KEY `host` (`host`)
KEY `host` (`host`),
KEY `date` (`date`),
KEY `idhost` (`idhost`),
CONSTRAINT `FK_newlogs_hosts` FOREIGN KEY (`idhost`) REFERENCES `hosts` (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='New logging table';
CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `LastHour` AS select `logger`.`newlogs`.`id` AS `id`,`logger`.`newlogs`.`date` AS `date`,`logger`.`newlogs`.`facility` AS `facility`,`logger`.`newlogs`.`level` AS `level`,`logger`.`newlogs`.`host` AS `host`,`logger`.`newlogs`.`program` AS `program`,`logger`.`newlogs`.`pid` AS `pid`,`logger`.`newlogs`.`message` AS `message` from `logger`.`newlogs` where (`logger`.`newlogs`.`date` >= (now() - interval 1 hour));
CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `Today` AS select `logger`.`newlogs`.`id` AS `id`,`logger`.`newlogs`.`date` AS `date`,`logger`.`newlogs`.`facility` AS `facility`,`logger`.`newlogs`.`level` AS `level`,`logger`.`newlogs`.`host` AS `host`,`logger`.`newlogs`.`program` AS `program`,`logger`.`newlogs`.`pid` AS `pid`,`logger`.`newlogs`.`message` AS `message` from `logger`.`newlogs` where (cast(now() as date) = cast(`logger`.`newlogs`.`date` as date));
CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `Week` AS select `logger`.`newlogs`.`id` AS `id`,`logger`.`newlogs`.`date` AS `date`,`logger`.`newlogs`.`facility` AS `facility`,`logger`.`newlogs`.`level` AS `level`,`logger`.`newlogs`.`host` AS `host`,`logger`.`newlogs`.`program` AS `program`,`logger`.`newlogs`.`pid` AS `pid`,`logger`.`newlogs`.`message` AS `message` from `logger`.`newlogs` where (`logger`.`newlogs`.`date` >= (now() - interval 168 hour));
-- Daten Export vom Benutzer nicht ausgewählt
-- Exportiere Struktur von View logger.Today
-- Erstelle temporäre Tabelle um View Abhängigkeiten zuvorzukommen
CREATE TABLE `Today` (
`id` BIGINT(20) NOT NULL,
`date` DATETIME NOT NULL,
`facility` ENUM('kern','user','mail','daemon','auth','syslog','lpr','news','uucp','authpriv','ftp','cron','local0','local1','local2','local3','local4','local5','local6','local7') NOT NULL COLLATE 'utf8_general_ci',
`level` ENUM('emerg','alert','crit','err','warning','notice','info','debug') NOT NULL COLLATE 'utf8_general_ci',
`host` VARCHAR(50) NOT NULL COLLATE 'utf8_general_ci',
`program` VARCHAR(50) NOT NULL COLLATE 'utf8_general_ci',
`pid` INT(10) UNSIGNED NOT NULL,
`message` TEXT NOT NULL COLLATE 'utf8_general_ci',
`idhost` INT(11) NULL
) ENGINE=MyISAM;
-- Exportiere Struktur von View logger.Week
-- Erstelle temporäre Tabelle um View Abhängigkeiten zuvorzukommen
CREATE TABLE `Week` (
`id` BIGINT(20) NOT NULL,
`date` DATETIME NOT NULL,
`facility` ENUM('kern','user','mail','daemon','auth','syslog','lpr','news','uucp','authpriv','ftp','cron','local0','local1','local2','local3','local4','local5','local6','local7') NOT NULL COLLATE 'utf8_general_ci',
`level` ENUM('emerg','alert','crit','err','warning','notice','info','debug') NOT NULL COLLATE 'utf8_general_ci',
`host` VARCHAR(50) NOT NULL COLLATE 'utf8_general_ci',
`program` VARCHAR(50) NOT NULL COLLATE 'utf8_general_ci',
`pid` INT(10) UNSIGNED NOT NULL,
`message` TEXT NOT NULL COLLATE 'utf8_general_ci',
`idhost` INT(11) NULL
) ENGINE=MyISAM;
-- Exportiere Struktur von View logger.LastHour
-- Entferne temporäre Tabelle und erstelle die eigentliche View
DROP TABLE IF EXISTS `LastHour`;
CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `LastHour` AS select `newlogs`.`id` AS `id`,`newlogs`.`date` AS `date`,`newlogs`.`facility` AS `facility`,`newlogs`.`level` AS `level`,`newlogs`.`host` AS `host`,`newlogs`.`program` AS `program`,`newlogs`.`pid` AS `pid`,`newlogs`.`message` AS `message`,`newlogs`.`idhost` AS `idhost` from `newlogs` where (`newlogs`.`date` >= (now() - interval 1 hour));
-- Exportiere Struktur von View logger.Today
-- Entferne temporäre Tabelle und erstelle die eigentliche View
DROP TABLE IF EXISTS `Today`;
CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `Today` AS select `newlogs`.`id` AS `id`,`newlogs`.`date` AS `date`,`newlogs`.`facility` AS `facility`,`newlogs`.`level` AS `level`,`newlogs`.`host` AS `host`,`newlogs`.`program` AS `program`,`newlogs`.`pid` AS `pid`,`newlogs`.`message` AS `message`,`newlogs`.`idhost` AS `idhost` from `newlogs` where (cast(now() as date) = cast(`newlogs`.`date` as date));
-- Exportiere Struktur von View logger.Week
-- Entferne temporäre Tabelle und erstelle die eigentliche View
DROP TABLE IF EXISTS `Week`;
CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `Week` AS select `newlogs`.`id` AS `id`,`newlogs`.`date` AS `date`,`newlogs`.`facility` AS `facility`,`newlogs`.`level` AS `level`,`newlogs`.`host` AS `host`,`newlogs`.`program` AS `program`,`newlogs`.`pid` AS `pid`,`newlogs`.`message` AS `message`,`newlogs`.`idhost` AS `idhost` from `newlogs` where (`newlogs`.`date` >= (now() - interval 168 hour));
/*!40101 SET SQL_MODE=IFNULL(@OLD_SQL_MODE, '') */;
/*!40014 SET FOREIGN_KEY_CHECKS=IF(@OLD_FOREIGN_KEY_CHECKS IS NULL, 1, @OLD_FOREIGN_KEY_CHECKS) */;
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;

+ 160
- 43
inc/lggr_class.php View File

@@ -5,15 +5,14 @@ class Lggr {
private $config=null;
private $db=null;
private $state=null;
private $perfTime=null;
private $perfCount=null;
private $cache=null;
private $aPerf=null;

function __construct(LggrState $state, AbstractConfig $config) {
$this->config = $config;
$this->state = $state;

$this->perfCount=0;
$this->perfTime=0;
$this->cache = new LggrCache();
$this->aPerf = array(); // of type LggrPerf objects

if(!$this->state->isLocalCall())
$this->checkSecurity();
@@ -43,17 +42,23 @@ class Lggr {
}

function getLevels() {
$this->perfCount++;
$startTime = microtime(true);
$perf = new LggrPerf();

$v = $this->getViewName();
$a = array();
$sql = "
SELECT level, COUNT(*) AS c FROM $v
GROUP BY level
ORDER BY c DESC
";

$a = $this->cache->retrieve("levels$v");
if(null != $a) {
return $a;
} // if
$a = array();

$perf->start($sql);

$res = $this->db->query($sql);
if(false === $res) {
throw new Exception($this->db->error);
@@ -72,22 +77,32 @@ ORDER BY c DESC
$level->f = round($f, 2);
} // foreach

$this->perfTime += microtime(true)-$startTime;
$perf->stop();
$this->aPerf[] = $perf;

$this->cache->store("levels$v", $a);
return $a;
} // function

function getServers() {
$this->perfCount++;
$startTime = microtime(true);
$perf = new LggrPerf();

$v = $this->getViewName();
$a = array();
$sql = "
SELECT host, COUNT(*) AS c FROM $v
GROUP BY host
ORDER BY c DESC
";
SELECT h.name as host, COUNT(*) AS c
FROM $v d
JOIN hosts h ON d.idhost=h.id
GROUP BY h.id
ORDER BY c DESC";

$a = $this->cache->retrieve("servers$v");
if(null != $a) {
return $a;
} // if
$a = array();

$perf->start($sql);

$res = $this->db->query($sql);
if(false === $res) {
@@ -107,14 +122,16 @@ ORDER BY c DESC
$host->f = round($f, 2);
} // foreach

$this->perfTime += microtime(true)-$startTime;
$perf->stop();
$this->aPerf[] = $perf;

$this->cache->store("servers$v", $a);
return $a;
} // function

function getLatest($from=0, $count=LggrState::PAGELEN) {
$this->perfCount += 2;
$startTime = microtime(true);
$perfSize = new LggrPerf();
$perfData = new LggrPerf();

$v = $this->getViewName();

@@ -124,27 +141,41 @@ SELECT * FROM $v
ORDER BY `date` DESC
LIMIT $from,$count";

$perfSize->start($sqlSize);
$this->getResultSize($sqlSize);
$perfSize->stop();

$perfData->start($sqlData);
$a = $this->sendResult($sqlData);
$perfData->stop();

$this->perfTime += microtime(true)-$startTime;
$this->aPerf[] = $perfSize;
$this->aPerf[] = $perfData;

return $a;
} // function

function getNewer($id) {
$perf = new LggrPerf();

$sqlData = "
SELECT * FROM LastHour
WHERE id>$id
ORDER BY `date` DESC
LIMIT " . LggrState::PAGELEN;

return $this->sendResult($sqlData);
$perf->start($sqlData);
$a = $this->sendResult($sqlData);
$perf->stop();

$this->aPerf[] = $perf;

return $a;
} // function

function getFromTo($from=0, $count=LggrState::PAGELEN) {
$this->perfCount += 2;
$startTime = microtime(true);
$perfSize = new LggrPerf();
$perfData = new LggrPerf();

$sFrom = $this->db->escape_string($this->state->getFrom());
$sTo = $this->db->escape_string($this->state->getTo());
@@ -159,17 +190,23 @@ WHERE `date` BETWEEN '$sFrom' AND '$sTo'
ORDER BY `date` DESC
LIMIT $from,$count";

$perfSize->start($sqlSize);
$this->getResultSize($sqlSize);
$perfSize->stop();

$perfData->start($sqlData);
$a = $this->sendResult($sqlData);
$perfData->stop();

$this->perfTime += microtime(true)-$startTime;
$this->aPerf[] = $perfSize;
$this->aPerf[] = $perfData;

return $a;
} // function

function getFiltered($host=null, $level=null, $from=0, $count=LggrState::PAGELEN) {
$this->perfCount += 2;
$startTime = microtime(true);
$perfSize = new LggrPerf();
$perfData = new LggrPerf();

$v = $this->getViewName();

@@ -193,17 +230,22 @@ LIMIT $from,$count";

$sqlData .= " ORDER BY `date` DESC LIMIT $from,$count";

$perfSize->start($sqlSize);
$this->getResultSize($sqlSize);
$perfSize->stop();

$perfData->start($sqlData);
$a = $this->sendResult($sqlData);
$perfData->stop();

$this->perfTime += microtime(true)-$startTime;
$this->aPerf[] = $perfSize;
$this->aPerf[] = $perfData;

return $a;
} // function

function getText($msg='', $prog='', $from=0, $count=LggrState::PAGELEN) {
$this->perfCount++;
$startTime = microtime(true);
$perf = new LggrPerf();

$v = $this->getViewName();
$sTmpMsg = $this->db->escape_string($msg);
@@ -224,42 +266,63 @@ WHERE $sWhere
ORDER BY `date` DESC
LIMIT $from,$count";

$perf->start($sql);
$a = $this->sendResult($sql);
$perf->stop();

$this->perfTime += microtime(true)-$startTime;
$this->aPerf[] = $perf;

return $a;
} // function

function getMessagesPerHour() {
$this->perfCount++;
$startTime = microtime(true);
$perf = new LggrPerf();

$sql = "
SELECT HOUR(TIME(`date`)) AS h, COUNT(*) AS c
FROM Today
GROUP BY h";

$a = $this->cache->retrieve('mph');
if(null != $a) {
return $a;
} // if

$perf->start($sql);
$a = $this->sendResult($sql);
$perf->stop();

$this->perfTime += microtime(true)-$startTime;
$this->aPerf[] = $perf;

$this->cache->store('mph', $a);
return $a;
} // function

function getStatistic() {
$perf = new LggrPerf();

$sql = "
SELECT COUNT(*) AS cnt, MIN(`date`) AS oldest
FROM newlogs
";
return $this->sendResult($sql);

$a = $this->cache->retrieve('stats');
if(null != $a) {
return $a;
} // if

$perf->start($sql);
$a = $this->sendResult($sql);
$perf->stop();

$this->aPerf[] = $perf;

$this->cache->store('stats', $a);
return $a;
} // function

/* delete anything older than maxage hours, or 4 weeks */
function purgeOldMessages($maxage=672) {
$this->perfCount++;
$startTime = microtime(true);

$sql = "
DELETE FROM newlogs
WHERE `date` < (NOW() - INTERVAL $maxage hour)
@@ -269,11 +332,68 @@ WHERE `date` < (NOW() - INTERVAL $maxage hour)
throw new Exception($this->db->error);
} // if

$this->perfTime += microtime(true)-$startTime;

return $this->db->affected_rows;
} // function

function normalizeHosts() {

// Find any new hostnames
$sql = "
SELECT newlogs.host
FROM newlogs
LEFT JOIN hosts ON hosts.name=newlogs.host
WHERE hosts.id IS NULL
GROUP BY newlogs.host";
$aEmpty = $this->sendResult($sql);
foreach($aEmpty as $o) {
$host = $o->host;
$host = $this->db->escape_string($host);

$sql = "INSERT INTO hosts (name) VALUES ('$host')";
$res = $this->db->query($sql);
if(false === $res) {
throw new Exception($this->db->error);
} // if
$id = $this->db->insert_id;
// echo "New $id for $host\n";

$sql = "UPDATE newlogs SET idhost=$id WHERE host='$host'";
$res = $this->db->query($sql);
if(false === $res) {
throw new Exception($this->db->error);
} // if
} // foreach

// read current list of hostnames and ids
$sql = "
SELECT *
FROM hosts";
$aTmp = $this->sendResult($sql);
$aHosts = array();
foreach($aTmp as $o) {
$hostId = $o->id;
$hostName = $o->name;
$aHosts[$hostName] = $hostId;
} // foreach

// search any new entry without hostid and update it
foreach($aHosts as $hostName => $hostId) {
// echo "updating $hostName to $hostId\n";
$hostName = $this->db->escape_string($hostName);
$sql = "
UPDATE newlogs
SET idhost=$hostId
WHERE idhost IS NULL
AND host='$hostName'
";
$res = $this->db->query($sql);
if(false === $res) {
throw new Exception($this->db->error);
} // if
} // foreach

} // function


private function getResultSize($sql) {
$res = $this->db->query($sql);
@@ -303,10 +423,7 @@ WHERE `date` < (NOW() - INTERVAL $maxage hour)
} // function

public function getPerf() {
return array(
'count' => $this->perfCount,
'time' => $this->perfTime
);
return $this->aPerf;
} // function

} // class

+ 51
- 0
inc/lggrcache_class.php View File

@@ -0,0 +1,51 @@
<?php

class LggrCache {
const MAXAGE = 300; // 5 minutes
private $cachepath=null;

function __construct() {
$this->cachepath = __DIR__ . '/../cache/';
} // constructor

public function store($key, $value) {
$fname = $this->getFilename($key);
$s = serialize($value);
file_put_contents($fname, $s);
} // function

public function retrieve($key) {
$fname = $this->getFilename($key);
if(file_exists($fname) && is_readable($fname)) {
$ts = filemtime($fname);
if(time() - $ts > self::MAXAGE) {
unlink($fname);
return null;
} else {
$s = file_get_contents($fname);
$a = unserialize($s);
return $a;
} // if
} else {
return null;
} // if
} // function

public function purge($key) {
$fname = $this->getFilename($key);
unlink($fname);
} // function


private function filterKey($key) {
$sTmp = str_replace(' ', '-', $key);
$sTmp = preg_replace('/[^A-Za-z0-9\-]/', '', $sTmp);
return $sTmp;
} // function

private function getFilename($key) {
return $this->cachepath . 'key_' . $this->filterKey($key) . '.data';
} // function

}


+ 37
- 0
inc/lggrperf_class.php View File

@@ -0,0 +1,37 @@
<?php

class LggrPerf {
private $tsStart=null;
private $tsEnd=null;
private $tsLen=null;
private $sQuery=null;

function __construct() {
} // constructor

public function start($sql) {
$this->sQuery = $sql;
$this->tsStart = microtime(true);
} // function

public function stop() {
$this->tsEnd = microtime(true);
$this->tsLen = $this->tsEnd - $this->tsStart;
} // function

public function getPerf() {
$a = array();

$a['time'] = $this->tsLen;
$a['query'] = $this->sQuery;

$this->logperf();

return $a;
} // function

private function logPerf() {
error_log("Perf " . $this->tsLen . ", query " . $this->sQuery);
} // function
}


+ 3
- 0
index.php View File

@@ -55,6 +55,7 @@ try {

$aEvents = $l->getFiltered($host, $level, $page*LggrState::PAGELEN, LggrState::PAGELEN);
$searchvalue='';
$searchvalueprog='';
$isSearch=false;
$sFilter='';
if($state->isHost())
@@ -67,6 +68,7 @@ try {
$aEvents = $l->getFromTo($page*LggrState::PAGELEN, LggrState::PAGELEN);
$sFilter = 'Filter by time range between <strong>' . htmlentities($state->getFrom()) . '</strong> and <strong>' . htmlentities($state->getTo()) . '</strong>';
$searchvalue='';
$searchvalueprog='';
$isSearch=false;

} else {
@@ -75,6 +77,7 @@ try {

$aEvents = $l->getLatest($page*LggrState::PAGELEN, LggrState::PAGELEN);
$searchvalue='';
$searchvalueprog='';
$isSearch=false;

} // if search

+ 10
- 1
tpl/foot.inc.php View File

@@ -1,7 +1,16 @@
<div class="container">
<hr>
<footer>
<p class="debugfooter"><?= $aPerf['count'] ?> queries in <?= $aPerf['time'] ?> seconds. Session: <?= $_COOKIE['PHPSESSID'] ?> by <?= htmlentities($_SERVER['REMOTE_USER']) ?></p>
<?php
$pCount = count($aPerf);
$pTime = 0;
foreach($aPerf as $perf) {
$aTmp = $perf->getPerf();

$pTime += $aTmp['time'];
} // foreach
?>
<p class="debugfooter"><?= $pCount ?> queries in <?= $pTime ?> seconds. Session: <?= $_COOKIE['PHPSESSID'] ?> by <?= htmlentities($_SERVER['REMOTE_USER']) ?></p>
<p>&copy; <a href="http://lggr.io" target="_blank">lggr.io</a> 2015</p>
</footer>
</div> <!-- /container -->

Loading…
Cancel
Save
Social stuff:
Mastodon