ElasticSearch - uax_url_email 토크나이저 [ 예제, 설명 ]

-- standard 토크나이저는 @ , /  등과 같은 특수문자는 제거하고 분리한다
-- 하지만 URL 등이 삽입된 데이터에서 이런 특수문자가 사라지면 안된다
-- 이를 방지하기 위해 사용 가능한 것이 UAX URL Email 토크나이저 이다


-- standard 토크나이저로 분리한 경우
GET _analyze
{
  "tokenizer": "standard",
  "text": "email address is my-name@email.com and website is https://www.elastic.co"
}


-- 이메일이 분리 되버림
{
  "tokens" : [
    {
      "token" : "email",
      "start_offset" : 0,
      "end_offset" : 5,
      "type" : "<ALPHANUM>",
      "position" : 0
    },
    {
      "token" : "address",
      "start_offset" : 6,
      "end_offset" : 13,
      "type" : "<ALPHANUM>",
      "position" : 1
    },
    {
      "token" : "is",
      "start_offset" : 14,
      "end_offset" : 16,
      "type" : "<ALPHANUM>",
      "position" : 2
    },
    {
      "token" : "my",
      "start_offset" : 17,
      "end_offset" : 19,
      "type" : "<ALPHANUM>",
      "position" : 3
    },
    {
      "token" : "name",
      "start_offset" : 20,
      "end_offset" : 24,
      "type" : "<ALPHANUM>",
      "position" : 4
    },
    {
      "token" : "email.com",
      "start_offset" : 25,
      "end_offset" : 34,
      "type" : "<ALPHANUM>",
      "position" : 5
    },
    {
      "token" : "and",
      "start_offset" : 35,
      "end_offset" : 38,
      "type" : "<ALPHANUM>",
      "position" : 6
    },
    {
      "token" : "website",
      "start_offset" : 39,
      "end_offset" : 46,
      "type" : "<ALPHANUM>",
      "position" : 7
    },
    {
      "token" : "is",
      "start_offset" : 47,
      "end_offset" : 49,
      "type" : "<ALPHANUM>",
      "position" : 8
    },
    {
      "token" : "https",
      "start_offset" : 50,
      "end_offset" : 55,
      "type" : "<ALPHANUM>",
      "position" : 9
    },
    {
      "token" : "www.elastic.co",
      "start_offset" : 58,
      "end_offset" : 72,
      "type" : "<ALPHANUM>",
      "position" : 10
    }
  ]
}



-- uax_url_email 토크나이저 사용
GET _analyze
{
  "tokenizer": "uax_url_email",
  "text": "email address is my-name@email.com and website is https://www.elastic.co"
}

-- email과 url이 깔끔하게 분리 되었다
{
  "tokens" : [
    {
      "token" : "email",
      "start_offset" : 0,
      "end_offset" : 5,
      "type" : "<ALPHANUM>",
      "position" : 0
    },
    {
      "token" : "address",
      "start_offset" : 6,
      "end_offset" : 13,
      "type" : "<ALPHANUM>",
      "position" : 1
    },
    {
      "token" : "is",
      "start_offset" : 14,
      "end_offset" : 16,
      "type" : "<ALPHANUM>",
      "position" : 2
    },
    {
      "token" : "my-name@email.com",
      "start_offset" : 17,
      "end_offset" : 34,
      "type" : "<EMAIL>",
      "position" : 3
    },
    {
      "token" : "and",
      "start_offset" : 35,
      "end_offset" : 38,
      "type" : "<ALPHANUM>",
      "position" : 4
    },
    {
      "token" : "website",
      "start_offset" : 39,
      "end_offset" : 46,
      "type" : "<ALPHANUM>",
      "position" : 5
    },
    {
      "token" : "is",
      "start_offset" : 47,
      "end_offset" : 49,
      "type" : "<ALPHANUM>",
      "position" : 6
    },
    {
      "token" : "https://www.elastic.co",
      "start_offset" : 50,
      "end_offset" : 72,
      "type" : "<URL>",
      "position" : 7
    }
  ]
}