1 files changed, 133 insertions, 0 deletions
diff --git a/IRCSharp.Tests/Tokenization/Tokenization.cs b/IRCSharp.Tests/Tokenization/Tokenization.cs
new file mode 100644
index 0000000..c4c5c5a
--- /dev/null
+++ b/IRCSharp.Tests/Tokenization/Tokenization.cs
@@ -0,0 +1,133 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace IRCTokens.Tests
+{
+    [TestClass]
+    public class Tokenization
+    {
+        [TestMethod]
+        public void TagsMissing()
+        {
+            var line = new Line("PRIVMSG #channel");
+            Assert.IsNull(line.Tags);
+        }
+
+        [TestMethod]
+        public void TagsMissingValue()
+        {
+            var line = new Line("@id= PRIVMSG #channel");
+            Assert.AreEqual(string.Empty, line.Tags["id"]);
+        }
+
+        [TestMethod]
+        public void TagsMissingEqual()
+        {
+            var line = new Line("@id PRIVMSG #channel");
+            Assert.AreEqual(string.Empty, line.Tags["id"]);
+        }
+
+        [TestMethod]
+        public void TagsUnescape()
+        {
+            var line = new Line(@"@id=1\\\:\r\n\s2 PRIVMSG #channel");
+            Assert.AreEqual("1\\;\r\n 2", line.Tags["id"]);
+        }
+
+        [TestMethod]
+        public void TagsOverlap()
+        {
+            var line = new Line(@"@id=1\\\s\\s PRIVMSG #channel");
+            Assert.AreEqual("1\\ \\s", line.Tags["id"]);
+        }
+
+        [TestMethod]
+        public void TagsLoneEndSlash()
+        {
+            var line = new Line("@id=1\\ PRIVMSG #channel");
+            Assert.AreEqual("1", line.Tags["id"]);
+        }
+
+        [TestMethod]
+        public void SourceWithoutTags()
+        {
+            var line = new Line(":nick!user@host PRIVMSG #channel");
+            Assert.AreEqual("nick!user@host", line.Source);
+        }
+
+        [TestMethod]
+        public void SourceWithTags()
+        {
+            var line = new Line("@id=123 :nick!user@host PRIVMSG #channel");
+            Assert.AreEqual("nick!user@host", line.Source);
+        }
+
+        [TestMethod]
+        public void SourceMissingWithoutTags()
+        {
+            var line = new Line("PRIVMSG #channel");
+            Assert.IsNull(line.Source);
+        }
+
+        [TestMethod]
+        public void SourceMissingWithTags()
+        {
+            var line = new Line("@id=123 PRIVMSG #channel");
+            Assert.IsNull(line.Source);
+        }
+
+        [TestMethod]
+        public void Command()
+        {
+            var line = new Line("privmsg #channel");
+            Assert.AreEqual("PRIVMSG", line.Command);
+        }
+
+        [TestMethod]
+        public void ParamsTrailing()
+        {
+            var line = new Line("PRIVMSG #channel :hello world");
+            CollectionAssert.AreEqual(new List<string> {"#channel", "hello world"}, line.Params);
+        }
+
+        [TestMethod]
+        public void ParamsOnlyTrailing()
+        {
+            var line = new Line("PRIVMSG :hello world");
+            CollectionAssert.AreEqual(new List<string> {"hello world"}, line.Params);
+        }
+
+        [TestMethod]
+        public void ParamsMissing()
+        {
+            var line = new Line("PRIVMSG");
+            Assert.AreEqual("PRIVMSG", line.Command);
+            CollectionAssert.AreEqual(new List<string>(), line.Params);
+        }
+
+        [TestMethod]
+        public void AllTokens()
+        {
+            var line = new Line("@id=123 :nick!user@host PRIVMSG #channel :hello world");
+            CollectionAssert.AreEqual(new Dictionary<string, string> {{"id", "123"}}, line.Tags);
+            Assert.AreEqual("nick!user@host", line.Source);
+            Assert.AreEqual("PRIVMSG", line.Command);
+            CollectionAssert.AreEqual(new List<string> {"#channel", "hello world"}, line.Params);
+        }
+
+        [TestMethod]
+        public void NulByte()
+        {
+            var decoder = new IRCTokens.StatefulDecoder();
+            var bytes = Encoding.UTF8.GetBytes(":nick!user@host PRIVMSG #channel :hello")
+                .Concat(Encoding.UTF8.GetBytes("\0"))
+                .Concat(Encoding.UTF8.GetBytes("world"))
+                .ToArray();
+            var line = decoder.Push(bytes, bytes.Length).First();
+            
+            CollectionAssert.AreEqual(new List<string> {"#channel", "hello"}, line.Params);
+        }
+    }
+}