From 736815ea831bc7fdecd0f9680f3218b70684b6ce Mon Sep 17 00:00:00 2001 From: Trolli Schmittlauch Date: Mon, 27 Jul 2020 21:49:42 +0200 Subject: [PATCH] normalise hastag unicode representation of incoming posts --- Hash2Pub.cabal | 2 +- src/Hash2Pub/PostService.hs | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Hash2Pub.cabal b/Hash2Pub.cabal index 3ca520e..56441ad 100644 --- a/Hash2Pub.cabal +++ b/Hash2Pub.cabal @@ -46,7 +46,7 @@ category: Network extra-source-files: CHANGELOG.md common deps - build-depends: base ^>=4.12.0.0, containers ^>=0.6.0.1, bytestring, utf8-string ^>=1.0.1.1, network ^>=2.8.0.1, time ^>=1.8.0.2, cmdargs ^>= 0.10, cryptonite ^>= 0.25, memory, async, stm, asn1-encoding, asn1-types, asn1-parse, publicsuffix, network-byte-order, safe, iproute, mtl, random, servant, servant-server, servant-client, warp, text, unordered-containers, hashable + build-depends: base ^>=4.12.0.0, containers ^>=0.6.0.1, bytestring, utf8-string ^>=1.0.1.1, network ^>=2.8.0.1, time ^>=1.8.0.2, cmdargs ^>= 0.10, cryptonite ^>= 0.25, memory, async, stm, asn1-encoding, asn1-types, asn1-parse, publicsuffix, network-byte-order, safe, iproute, mtl, random, servant, servant-server, servant-client, warp, text, unordered-containers, hashable, unicode-transforms ghc-options: -Wall diff --git a/src/Hash2Pub/PostService.hs b/src/Hash2Pub/PostService.hs index fc3e5e8..e44c8c6 100644 --- a/src/Hash2Pub/PostService.hs +++ b/src/Hash2Pub/PostService.hs @@ -21,6 +21,8 @@ import qualified Data.HashSet as HSet import Data.Maybe (fromMaybe) import Data.String (fromString) import qualified Data.Text.Lazy as Txt +import Data.Text.Normalize (NormalizationMode (NFC), + normalize) import Data.Time.Clock.POSIX import System.Random @@ -135,7 +137,7 @@ relayInbox :: PostService d -> Txt.Text -> Handler NoContent relayInbox serv post = do -- extract contained hashtags let - containedTags = fmap Txt.tail . filter ((==) '#' . Txt.head) . Txt.words $ post + containedTags = fmap (Txt.fromStrict . normalize NFC . Txt.toStrict . Txt.tail) . filter ((==) '#' . Txt.head) . Txt.words $ post -- generate post ID postId <- liftIO $ Txt.pack . show <$> (randomRIO (0, 2^128-1) :: IO Integer) -- add ID to own posts