New Release

Nicholas Bergsen · Nicholas Bergsen · commit 3bbe3563a657 · 2022-01-03T16:25:27.000+09:00
diff --git a/Nick.RobotsParser/Const.cs b/Nick.RobotsParser/Const.cs
@@ -2,7 +2,7 @@
 using System.Collections.Generic;
 using System.Text;
 
-namespace RobotsSharpParser
+namespace RobotsParser
 {
     public class Const
     {
diff --git a/Nick.RobotsParser/NamespaceIgnoreantXmlTextReader.cs b/Nick.RobotsParser/NamespaceIgnoreantXmlTextReader.cs
@@ -2,7 +2,7 @@
 using System.Threading.Tasks;
 using System.Xml;
 
-namespace RobotsSharpParser
+namespace RobotsParser
 {
     //This class is used to solve the issue of some sites using http and others using https
     //in their sitemap files.
diff --git a/Nick.RobotsParser/Nick.RobotsParser.csproj b/Nick.RobotsParser/Nick.RobotsParser.csproj
@@ -1,14 +1,17 @@
-<Project Sdk="Microsoft.NET.Sdk">
+﻿<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>net6.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
-    <PackageId>RobotsSharpParser</PackageId>
+    <PackageId>RobotsParser</PackageId>
     <Version>2.0.0</Version>
     <Authors>Nicholas Bergesen</Authors>
     <Company>Nicholas Bergesen</Company>
-    <PackageReleaseNotes>Update to net6 and minor bug fixes.</PackageReleaseNotes>
+    <PackageReleaseNotes>
+	    - Handle all types of compression not just GZip	
+		- Improve sitemap parsing robustness.
+	</PackageReleaseNotes>
     <PackageTags>robots,parse robots,web crawling,robots.txt,web scraping,spider,sitemap,sitemap parse</PackageTags>
     <Description>
       This client library enables working with Robots.txt.
diff --git a/Nick.RobotsParser/RobotsNotloadedException.cs b/Nick.RobotsParser/RobotsNotloadedException.cs
@@ -1,7 +1,7 @@
 using System;
 using System.Runtime.Serialization;
 
-namespace RobotsSharpParser
+namespace RobotsParser
 {
     public class RobotsNotloadedException : Exception
     {
diff --git a/Nick.RobotsParser/RobotsParser.cs b/Nick.RobotsParser/RobotsParser.cs
@@ -5,7 +5,7 @@
 using System.Xml.Linq;
 using System.Xml.Serialization;
 
-namespace RobotsSharpParser
+namespace RobotsParser
 {
     public interface IRobots
     {
diff --git a/Nick.RobotsParser/Useragent.cs b/Nick.RobotsParser/Useragent.cs
@@ -2,7 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 
-namespace RobotsSharpParser
+namespace RobotsParser
 {
     public class Useragent
     {
diff --git a/Nick.RobotsParser/siteindex.cs b/Nick.RobotsParser/siteindex.cs
diff --git a/Nick.RobotsParser/siteindex.xsd b/Nick.RobotsParser/siteindex.xsd
diff --git a/Nick.RobotsParser/sitemap.cs b/Nick.RobotsParser/sitemap.cs
diff --git a/Nick.RobotsParser/sitemap.xsd b/Nick.RobotsParser/sitemap.xsd
diff --git a/UnitTests/Tests.cs b/UnitTests/Tests.cs
@@ -1,7 +1,7 @@
 using NUnit.Framework;
+using RobotsParser;
 using System.IO;
 using System.Linq;
-using RobotsSharpParser;
 
 namespace UnitTests;
 
diff --git a/UnitTests/UnitTests.csproj b/UnitTests/UnitTests.csproj
@@ -14,10 +14,6 @@
     <PackageReference Include="coverlet.collector" Version="3.1.0" />
   </ItemGroup>
 
-  <ItemGroup>
-    <ProjectReference Include="..\Robots\Robots.csproj" />
-  </ItemGroup>
-
   <ItemGroup>
     <Content Include="lootRobots.txt">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
@@ -26,4 +22,8 @@
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </Content>
   </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Nick.RobotsParser\Nick.RobotsParser.csproj" />
+  </ItemGroup>
 </Project>
diff --git a/robots.sln b/robots.sln
@@ -1,28 +1,31 @@
-﻿
-Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 16
-VisualStudioVersion = 16.0.30114.105
-MinimumVisualStudioVersion = 10.0.40219.1
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Robots", "Robots\Robots.csproj", "{69399400-9965-4A5A-9268-75C0C4EE7180}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "UnitTests", "UnitTests\UnitTests.csproj", "{098AB90C-E79B-403B-A24C-AA3F6D85D2B2}"
-EndProject
-Global
-	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|Any CPU = Debug|Any CPU
-		Release|Any CPU = Release|Any CPU
-	EndGlobalSection
-	GlobalSection(SolutionProperties) = preSolution
-		HideSolutionNode = FALSE
-	EndGlobalSection
-	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{69399400-9965-4A5A-9268-75C0C4EE7180}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{69399400-9965-4A5A-9268-75C0C4EE7180}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{69399400-9965-4A5A-9268-75C0C4EE7180}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{69399400-9965-4A5A-9268-75C0C4EE7180}.Release|Any CPU.Build.0 = Release|Any CPU
-		{098AB90C-E79B-403B-A24C-AA3F6D85D2B2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{098AB90C-E79B-403B-A24C-AA3F6D85D2B2}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{098AB90C-E79B-403B-A24C-AA3F6D85D2B2}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{098AB90C-E79B-403B-A24C-AA3F6D85D2B2}.Release|Any CPU.Build.0 = Release|Any CPU
-	EndGlobalSection
-EndGlobal
+﻿
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.0.32014.148
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "UnitTests", "UnitTests\UnitTests.csproj", "{098AB90C-E79B-403B-A24C-AA3F6D85D2B2}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Nick.RobotsParser", "Nick.RobotsParser\Nick.RobotsParser.csproj", "{5125D1EF-6709-4683-B9D4-E8DF518FFEA9}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{098AB90C-E79B-403B-A24C-AA3F6D85D2B2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{098AB90C-E79B-403B-A24C-AA3F6D85D2B2}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{098AB90C-E79B-403B-A24C-AA3F6D85D2B2}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{098AB90C-E79B-403B-A24C-AA3F6D85D2B2}.Release|Any CPU.Build.0 = Release|Any CPU
+		{5125D1EF-6709-4683-B9D4-E8DF518FFEA9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5125D1EF-6709-4683-B9D4-E8DF518FFEA9}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5125D1EF-6709-4683-B9D4-E8DF518FFEA9}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5125D1EF-6709-4683-B9D4-E8DF518FFEA9}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {5752D947-5693-4A26-9ECF-6F6878359555}
+	EndGlobalSection
+EndGlobal

Original file line number	Diff line number	Diff line change
`@@ -2,7 +2,7 @@`
`2`	`2`	`using System.Collections.Generic;`
`3`	`3`	`using System.Text;`
`4`	`4`
`5`		`-namespace RobotsSharpParser`
	`5`	`+namespace RobotsParser`
`6`	`6`	`{`
`7`	`7`	`public class Const`
`8`	`8`	`{`
Original file line number	Diff line number	Diff line change
`@@ -2,7 +2,7 @@`
`2`	`2`	`using System.Threading.Tasks;`
`3`	`3`	`using System.Xml;`
`4`	`4`
`5`		`-namespace RobotsSharpParser`
	`5`	`+namespace RobotsParser`
`6`	`6`	`{`
`7`	`7`	`//This class is used to solve the issue of some sites using http and others using https`
`8`	`8`	`//in their sitemap files.`
Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,7 @@`
`1`	`1`	`using System;`
`2`	`2`	`using System.Runtime.Serialization;`
`3`	`3`
`4`		`-namespace RobotsSharpParser`
	`4`	`+namespace RobotsParser`
`5`	`5`	`{`
`6`	`6`	`public class RobotsNotloadedException : Exception`
`7`	`7`	`{`
Original file line number	Diff line number	Diff line change
`@@ -5,7 +5,7 @@`
`5`	`5`	`using System.Xml.Linq;`
`6`	`6`	`using System.Xml.Serialization;`
`7`	`7`
`8`		`-namespace RobotsSharpParser`
	`8`	`+namespace RobotsParser`
`9`	`9`	`{`
`10`	`10`	`public interface IRobots`
`11`	`11`	`{`
Original file line number	Diff line number	Diff line change
`@@ -2,7 +2,7 @@`
`2`	`2`	`using System.Collections.Generic;`
`3`	`3`	`using System.Linq;`
`4`	`4`
`5`		`-namespace RobotsSharpParser`
	`5`	`+namespace RobotsParser`
`6`	`6`	`{`
`7`	`7`	`public class Useragent`
`8`	`8`	`{`