Code Coverage |
||||||||||
Classes and Traits |
Functions and Methods |
Lines |
||||||||
| Total | |
100.00% |
1 / 1 |
|
100.00% |
9 / 9 |
CRAP | |
100.00% |
39 / 39 |
| TokenCountVectorizer | |
100.00% |
1 / 1 |
|
100.00% |
9 / 9 |
19 | |
100.00% |
39 / 39 |
| __construct | |
100.00% |
1 / 1 |
1 | |
100.00% |
5 / 5 |
|||
| transform | |
100.00% |
1 / 1 |
2 | |
100.00% |
4 / 4 |
|||
| getVocabulary | |
100.00% |
1 / 1 |
1 | |
100.00% |
1 / 1 |
|||
| transformSample | |
100.00% |
1 / 1 |
3 | |
100.00% |
9 / 9 |
|||
| getTokenIndex | |
100.00% |
1 / 1 |
2 | |
100.00% |
3 / 3 |
|||
| updateFrequency | |
100.00% |
1 / 1 |
2 | |
100.00% |
4 / 4 |
|||
| checkDocumentFrequency | |
100.00% |
1 / 1 |
3 | |
100.00% |
5 / 5 |
|||
| unsetBeyondMinimum | |
100.00% |
1 / 1 |
2 | |
100.00% |
3 / 3 |
|||
| getBeyondMinimumIndexes | |
100.00% |
1 / 1 |
3 | |
100.00% |
5 / 5 |
|||
| <?php | |
| declare (strict_types = 1); | |
| namespace Phpml\FeatureExtraction; | |
| use Phpml\Tokenization\Tokenizer; | |
| class TokenCountVectorizer implements Vectorizer | |
| { | |
| /** | |
| * @var Tokenizer | |
| */ | |
| private $tokenizer; | |
| /** | |
| * @var float | |
| */ | |
| private $minDF; | |
| /** | |
| * @var array | |
| */ | |
| private $vocabulary; | |
| /** | |
| * @var array | |
| */ | |
| private $frequencies; | |
| /** | |
| * @param Tokenizer $tokenizer | |
| * @param float $minDF | |
| */ | |
| public function __construct(Tokenizer $tokenizer, float $minDF = 0) | |
| { | |
| $this->tokenizer = $tokenizer; | |
| $this->minDF = $minDF; | |
| $this->vocabulary = []; | |
| $this->frequencies = []; | |
| } | |
| /** | |
| * @param array $samples | |
| * | |
| * @return array | |
| */ | |
| public function transform(array $samples): array | |
| { | |
| foreach ($samples as $index => $sample) { | |
| $samples[$index] = $this->transformSample($sample); | |
| } | |
| $samples = $this->checkDocumentFrequency($samples); | |
| return $samples; | |
| } | |
| /** | |
| * @return array | |
| */ | |
| public function getVocabulary() | |
| { | |
| return array_flip($this->vocabulary); | |
| } | |
| /** | |
| * @param string $sample | |
| * | |
| * @return array | |
| */ | |
| private function transformSample(string $sample) | |
| { | |
| $counts = []; | |
| $tokens = $this->tokenizer->tokenize($sample); | |
| foreach ($tokens as $token) { | |
| $index = $this->getTokenIndex($token); | |
| $this->updateFrequency($token); | |
| if (!isset($counts[$index])) { | |
| $counts[$index] = 0; | |
| } | |
| ++$counts[$index]; | |
| } | |
| return $counts; | |
| } | |
| /** | |
| * @param string $token | |
| * | |
| * @return mixed | |
| */ | |
| private function getTokenIndex(string $token) | |
| { | |
| if (!isset($this->vocabulary[$token])) { | |
| $this->vocabulary[$token] = count($this->vocabulary); | |
| } | |
| return $this->vocabulary[$token]; | |
| } | |
| /** | |
| * @param string $token | |
| */ | |
| private function updateFrequency(string $token) | |
| { | |
| if (!isset($this->frequencies[$token])) { | |
| $this->frequencies[$token] = 0; | |
| } | |
| ++$this->frequencies[$token]; | |
| } | |
| /** | |
| * @param array $samples | |
| * | |
| * @return array | |
| */ | |
| private function checkDocumentFrequency(array $samples) | |
| { | |
| if ($this->minDF > 0) { | |
| $beyondMinimum = $this->getBeyondMinimumIndexes(count($samples)); | |
| foreach ($samples as $index => $sample) { | |
| $samples[$index] = $this->unsetBeyondMinimum($sample, $beyondMinimum); | |
| } | |
| } | |
| return $samples; | |
| } | |
| /** | |
| * @param array $sample | |
| * @param array $beyondMinimum | |
| * | |
| * @return array | |
| */ | |
| private function unsetBeyondMinimum(array $sample, array $beyondMinimum) | |
| { | |
| foreach ($beyondMinimum as $index) { | |
| unset($sample[$index]); | |
| } | |
| return $sample; | |
| } | |
| /** | |
| * @param int $samplesCount | |
| * | |
| * @return array | |
| */ | |
| private function getBeyondMinimumIndexes(int $samplesCount) | |
| { | |
| $indexes = []; | |
| foreach ($this->frequencies as $token => $frequency) { | |
| if (($frequency / $samplesCount) < $this->minDF) { | |
| $indexes[] = $this->getTokenIndex($token); | |
| } | |
| } | |
| return $indexes; | |
| } | |
| } |