PHP Backend Application(六)Retry Function, PHPUNIT ENV and Async Redis and Large
PHP Backend Application(6)Retry Function, PHPUNIT ENV and Async Redis and Large
PHP Backend Application(6)Retry Function, PHPUNIT ENV and Async Redis and Large File
1 Retry Functions in PHP
/**
* post json params to prediction
* @param string $path
* @param array $params, format will be ['key1' => 'value1', 'key2' => 'value2',]
* @return response
*/
public function post2Prediction($path, $params)
{
return $this->retry(function() use ($path, $params){
$response = $this->predictionClient->request('POST', $path, [
'json' => $params
]);
return $response;
}, 10, 3);
}
/**
* retry many times
* @param function $f
* @param number $delay
* @param number $retryies
*/
private function retry($f, $delay = 10, $retryies = 3){
$logger = $this->ioc->getService("logger");
try {
return $f();
}catch(Exception $e){
if($retryies > 0){
sleep($delay);
return retry($f, $delay, $retryies - 1);
}else{
$logger->error(\GuzzleHttp\Psr7\str($e->getRequest()));
if ($e->hasResponse()) {
$logger->error(\GuzzleHttp\Psr7\str($e->getResponse()));
}
}
}
}
2 phpunit Running Env
http://elnur.pro/using-environment-variables-to-add-flexibility-to-phpunit-tests/
Add the environment parameters in phpunit.xml, it works in Eclipse env
<phpunit bootstrap="tests/bootstrap_test.php">
<php>
<env name="RUNNING_ENV" value="test"/>
</php>
<testsuites>
<testsuite name="unitsuite">
<directory>tests</directory>
</testsuite>
</testsuites>
</phpunit>
In the command line,
RUNNING_ENV=test phpunit --bootstrap vendor/autoload.php tests/JobProducerPHP/JobsXMLParserTest
RUNNING_ENV=test php src/SuperStarProducer.php 12001 '/Users/carl/company/code/jobs-producer/data/12001.xml'
3 Object Inheritance in PHP
http://stackoverflow.com/questions/11237511/multiple-ways-of-calling-parent-method-in-php
http://php.net/manual/en/language.oop5.inheritance.php
Abstract Base class
<?php
namespace JobProducerPHP;
require __DIR__.'/../../vendor/autoload.php';
abstract class Import
{
protected $ioc = null;
public function __construct($ioc)
{
$this->ioc = $ioc;
$logger = $this->ioc->getService("logger");
$config = $this->ioc->getService("config");
}
public function printInfo(){
$logger = $this->ioc->getService("logger");
$logger->debug("calling method in parent class.");
}
public function printInfo2(){
$logger = $this->ioc->getService("logger");
$logger->debug("calling method2 in parent class.");
}
}
?>
Sub Class had the actually implementation
<?php
namespace JobProducerPHP;
require __DIR__.'/../../vendor/autoload.php';
class ImportIndeedFmt extends Import
{
public function printInfo2(){
$logger = $this->ioc->getService("logger");
$logger->debug("before calling parent");
$this->printInfo();
parent::printInfo2();
$logger->debug("after calling parent");
}
}
4 Predis-Async
https://github.com/nrk/predis-async/blob/master/examples/execute_commands.php
dependency
"predis/predis-async": "dev-master"
Async RedisClient, it works pretty well
<?php
namespace JobProducerPHP;
require __DIR__.'/../../vendor/autoload.php';
use \Predis\Async\Client;
class RedisClient
{
private $client = null;
private $ioc = null;
public function __construct($ioc)
{
$this->ioc = $ioc;
$logger = $this->ioc->getService("logger");
$config = $this->ioc->getService("config");
$logger->info("==============Redis config start ==============");
$logger->info("redisHost = " . $config['redisHost']);
$logger->info("redisPort = " . $config['redisPort']);
$logger->info("===============================================");
try
{
$this->client = new Client('tcp://' . $config['redisHost'] . ":" . $config['redisPort']);
$logger->debug("Successfully set up Redis");
}
catch (Exception $e)
{
$logger->error("Couldn't connected to Redis");
$logger->error($e->getMessage());
}
}
/**
*
* @param array $messages format eg: array("key"=>"value", ...)
*/
public function setMessageBatch($messages)
{
$client = $this->client;
$logger = $this->ioc->getService("logger");
$queueClient = $this->ioc->getService("queueClient");
//send to redis
foreach($messages as $key => $value){
$client->connect(function ($client) use ($key, $value, $logger){
$logger->debug("connect to Redis success. 001");
$logger->debug("sending message for {$key} => {$value} 002");
$client->set($key, $value, function($response, $client) use ($logger, $key){
$logger->debug("response from server " . $response . " 003");
$client->disconnect();
});
});
$client->getEventLoop()->run();
$logger->debug("main thread runs here 004");
}
$logger->debug("main thread runs here 005");
//send to SQS
$keys = array_keys($messages);
$queueClient->sendMessageBatch($keys);
}
}
?>
5 PHP handle Large File
public function regexMatch($line){
return preg_match('</job>', $line);
}
public function splitLargeFile($filePath, $sourceID)
{
try{
$outputSize = 0;
$fileCount = 0;
$outputFile = null;
foreach( new SplFileObject('/data/1052.xml') as $line){
if($outputSize === 0){
$outputFileName = "/data/1052_split/{$fileCount}.xml";
$outputFile = fopen($outputFileName, "w");
}
$outputSize += strlen($line);
fwrite($outputFile, $line . PHP_EOL);
if($this->regexMatch($line) && $outputSize >= 100000000){
fclose($outputFile);
$outputSize = 0;
$fileCount++;
}
}
if($outputSize > 0){
fclose($outputFile);
}
}catch (Exception $e){
echo $e->getMessage();
}
}
PHP RegexUtil
public function regexMatchJob($line){
return preg_match('/<\/job>/i', $line);
}
public function extractReferenceNumber($line){
$result = '';
preg_match('/<[Rr]eferencenumber>(.*?)<\/[Rr]eferencenumber>/i', $line, $matches);
if(!empty($matches) && count($matches) > 1){
$result = $matches[1];
$result = preg_replace('/\]\]>/i','', $result);
$result = preg_replace('/<!\[CDATA\[/i','', $result);
}
return $result;
}
References:
retry system
https://gist.github.com/mudge/5948769
SOLR join
http://stackoverflow.com/questions/12665797/is-solr-4-0-capable-of-using-join-for-multiple-core
PRedis
https://github.com/phpredis/phpredis
Redis
http://redis.io/commands#sorted_set
PHP Backend Application(6)Retry Function, PHPUNIT ENV and Async Redis and Large File
1 Retry Functions in PHP
/**
* post json params to prediction
* @param string $path
* @param array $params, format will be ['key1' => 'value1', 'key2' => 'value2',]
* @return response
*/
public function post2Prediction($path, $params)
{
return $this->retry(function() use ($path, $params){
$response = $this->predictionClient->request('POST', $path, [
'json' => $params
]);
return $response;
}, 10, 3);
}
/**
* retry many times
* @param function $f
* @param number $delay
* @param number $retryies
*/
private function retry($f, $delay = 10, $retryies = 3){
$logger = $this->ioc->getService("logger");
try {
return $f();
}catch(Exception $e){
if($retryies > 0){
sleep($delay);
return retry($f, $delay, $retryies - 1);
}else{
$logger->error(\GuzzleHttp\Psr7\str($e->getRequest()));
if ($e->hasResponse()) {
$logger->error(\GuzzleHttp\Psr7\str($e->getResponse()));
}
}
}
}
2 phpunit Running Env
http://elnur.pro/using-environment-variables-to-add-flexibility-to-phpunit-tests/
Add the environment parameters in phpunit.xml, it works in Eclipse env
<phpunit bootstrap="tests/bootstrap_test.php">
<php>
<env name="RUNNING_ENV" value="test"/>
</php>
<testsuites>
<testsuite name="unitsuite">
<directory>tests</directory>
</testsuite>
</testsuites>
</phpunit>
In the command line,
RUNNING_ENV=test phpunit --bootstrap vendor/autoload.php tests/JobProducerPHP/JobsXMLParserTest
RUNNING_ENV=test php src/SuperStarProducer.php 12001 '/Users/carl/company/code/jobs-producer/data/12001.xml'
3 Object Inheritance in PHP
http://stackoverflow.com/questions/11237511/multiple-ways-of-calling-parent-method-in-php
http://php.net/manual/en/language.oop5.inheritance.php
Abstract Base class
<?php
namespace JobProducerPHP;
require __DIR__.'/../../vendor/autoload.php';
abstract class Import
{
protected $ioc = null;
public function __construct($ioc)
{
$this->ioc = $ioc;
$logger = $this->ioc->getService("logger");
$config = $this->ioc->getService("config");
}
public function printInfo(){
$logger = $this->ioc->getService("logger");
$logger->debug("calling method in parent class.");
}
public function printInfo2(){
$logger = $this->ioc->getService("logger");
$logger->debug("calling method2 in parent class.");
}
}
?>
Sub Class had the actually implementation
<?php
namespace JobProducerPHP;
require __DIR__.'/../../vendor/autoload.php';
class ImportIndeedFmt extends Import
{
public function printInfo2(){
$logger = $this->ioc->getService("logger");
$logger->debug("before calling parent");
$this->printInfo();
parent::printInfo2();
$logger->debug("after calling parent");
}
}
4 Predis-Async
https://github.com/nrk/predis-async/blob/master/examples/execute_commands.php
dependency
"predis/predis-async": "dev-master"
Async RedisClient, it works pretty well
<?php
namespace JobProducerPHP;
require __DIR__.'/../../vendor/autoload.php';
use \Predis\Async\Client;
class RedisClient
{
private $client = null;
private $ioc = null;
public function __construct($ioc)
{
$this->ioc = $ioc;
$logger = $this->ioc->getService("logger");
$config = $this->ioc->getService("config");
$logger->info("==============Redis config start ==============");
$logger->info("redisHost = " . $config['redisHost']);
$logger->info("redisPort = " . $config['redisPort']);
$logger->info("===============================================");
try
{
$this->client = new Client('tcp://' . $config['redisHost'] . ":" . $config['redisPort']);
$logger->debug("Successfully set up Redis");
}
catch (Exception $e)
{
$logger->error("Couldn't connected to Redis");
$logger->error($e->getMessage());
}
}
/**
*
* @param array $messages format eg: array("key"=>"value", ...)
*/
public function setMessageBatch($messages)
{
$client = $this->client;
$logger = $this->ioc->getService("logger");
$queueClient = $this->ioc->getService("queueClient");
//send to redis
foreach($messages as $key => $value){
$client->connect(function ($client) use ($key, $value, $logger){
$logger->debug("connect to Redis success. 001");
$logger->debug("sending message for {$key} => {$value} 002");
$client->set($key, $value, function($response, $client) use ($logger, $key){
$logger->debug("response from server " . $response . " 003");
$client->disconnect();
});
});
$client->getEventLoop()->run();
$logger->debug("main thread runs here 004");
}
$logger->debug("main thread runs here 005");
//send to SQS
$keys = array_keys($messages);
$queueClient->sendMessageBatch($keys);
}
}
?>
5 PHP handle Large File
public function regexMatch($line){
return preg_match('</job>', $line);
}
public function splitLargeFile($filePath, $sourceID)
{
try{
$outputSize = 0;
$fileCount = 0;
$outputFile = null;
foreach( new SplFileObject('/data/1052.xml') as $line){
if($outputSize === 0){
$outputFileName = "/data/1052_split/{$fileCount}.xml";
$outputFile = fopen($outputFileName, "w");
}
$outputSize += strlen($line);
fwrite($outputFile, $line . PHP_EOL);
if($this->regexMatch($line) && $outputSize >= 100000000){
fclose($outputFile);
$outputSize = 0;
$fileCount++;
}
}
if($outputSize > 0){
fclose($outputFile);
}
}catch (Exception $e){
echo $e->getMessage();
}
}
PHP RegexUtil
public function regexMatchJob($line){
return preg_match('/<\/job>/i', $line);
}
public function extractReferenceNumber($line){
$result = '';
preg_match('/<[Rr]eferencenumber>(.*?)<\/[Rr]eferencenumber>/i', $line, $matches);
if(!empty($matches) && count($matches) > 1){
$result = $matches[1];
$result = preg_replace('/\]\]>/i','', $result);
$result = preg_replace('/<!\[CDATA\[/i','', $result);
}
return $result;
}
References:
retry system
https://gist.github.com/mudge/5948769
SOLR join
http://stackoverflow.com/questions/12665797/is-solr-4-0-capable-of-using-join-for-multiple-core
PRedis
https://github.com/phpredis/phpredis
Redis
http://redis.io/commands#sorted_set