Merge pull request #42 from tsjason/main

Add some efficiencies to prevent unnecessary requests
spatie · Apr 19, 2024 · a676cac · a676cac
2 parents c59dc4e + ee01770
commit a676cac
Show file tree

Hide file tree

Showing 2 changed files with 39 additions and 10 deletions.
diff --git a/src/Robots.php b/src/Robots.php
@@ -8,16 +8,22 @@ class Robots
 
     public function __construct(
         protected string | null $userAgent = null,
-        string | null $source = null,
+        RobotsTxt | string | null $source = null,
     ) {
-        $this->robotsTxt = $source
-            ? RobotsTxt::readFrom($source)
-            : null;
+        if ($source instanceof RobotsTxt) {
+            $this->robotsTxt = $source;
+        } elseif (is_string($source)) {
+            $this->robotsTxt = RobotsTxt::readFrom($source);
+        } else {
+            $this->robotsTxt = null;
+        }
     }
 
-    public function withTxt(string $source): self
+    public function withTxt(RobotsTxt | string $source): self
     {
-        $this->robotsTxt = RobotsTxt::readFrom($source);
+        $this->robotsTxt = $source instanceof RobotsTxt
+                                ? $source
+                                : RobotsTxt::readFrom($source);
 
         return $this;
     }
@@ -33,17 +39,29 @@ public function mayIndex(string $url, string $userAgent = null): bool
 
         $robotsTxt = $this->robotsTxt ?? RobotsTxt::create($this->createRobotsUrl($url));
 
+        $content = @file_get_contents($url);
+
+        if ($content === false) {
+            throw new InvalidArgumentException("Could not read url `{$url}`");
+        }
+
         return
             $robotsTxt->allows($url, $userAgent)
-            && RobotsMeta::readFrom($url)->mayIndex()
-            && RobotsHeaders::readFrom($url)->mayIndex();
+            && RobotsMeta::create($content)->mayIndex()
+            && RobotsHeaders::create($http_response_header ?? [])->mayIndex();
     }
 
     public function mayFollowOn(string $url): bool
     {
+        $content = @file_get_contents($url);
+
+        if ($content === false) {
+            throw new InvalidArgumentException("Could not read url `{$url}`");
+        }
+
         return
-            RobotsMeta::readFrom($url)->mayFollow()
-            && RobotsHeaders::readFrom($url)->mayFollow();
+            RobotsMeta::create($content)->mayFollow()
+            && RobotsHeaders::create($http_response_header ?? [])->mayFollow();
     }
 
     protected function createRobotsUrl(string $url): string

diff --git a/tests/RobotsTest.php b/tests/RobotsTest.php
@@ -3,6 +3,7 @@
 namespace Spatie\Robots\Tests;
 
 use Spatie\Robots\Robots;
+use Spatie\Robots\RobotsTxt;
 
 class RobotsTest extends TestCase
 {
@@ -15,6 +16,16 @@ public function test()
         $this->assertTrue($robots->mayIndex('/'));
     }
 
+    /** @test */
+    public function it_can_be_created_with_a_robots_txt_object()
+    {
+        $robotsTxt = RobotsTxt::create(__DIR__.'/data/robots.txt');
+        $robots = Robots::create()
+            ->withTxt($robotsTxt);
+
+        $this->assertTrue($robots->mayIndex('/'));
+    }
+
     /** @test */
     public function it_return_true_on_source_string()
     {