Merge pull request #18 from DanielSWolf/feature/utf-8
Full Unicode support by using UTF-8 throughout
This commit is contained in:
Daniel Wolf
GitHub
commit
4de8f3d18e
|
|
@ -1,7 +1,7 @@
|
|||
cmake_minimum_required(VERSION 3.2)
|
||||
|
||||
# Support legacy OS X versions
|
||||
set(CMAKE_OSX_DEPLOYMENT_TARGET "10.7" CACHE STRING "Minimum OS X deployment version")
|
||||
set(CMAKE_OSX_DEPLOYMENT_TARGET "10.8" CACHE STRING "Minimum OS X deployment version")
|
||||
|
||||
set(appName "Rhubarb Lip Sync")
|
||||
set(appVersionMajor 1)
|
||||
|
|
@ -49,6 +49,11 @@ elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
|
|||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /utf-8")
|
||||
endif()
|
||||
|
||||
# Use UTF-8 throughout
|
||||
if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
|
||||
add_compile_options("/utf-8")
|
||||
endif()
|
||||
|
||||
if(${UNIX})
|
||||
add_definitions(-DHAVE_UNISTD_H)
|
||||
endif()
|
||||
|
|
@ -214,6 +219,24 @@ target_include_directories(flite SYSTEM PUBLIC
|
|||
target_compile_options(flite PRIVATE ${disableWarningsFlags})
|
||||
set_target_properties(flite PROPERTIES FOLDER lib)
|
||||
|
||||
# ... UTF8-CPP
|
||||
add_library(utfcpp
|
||||
lib/header-only.c
|
||||
lib/utfcpp-2.3.5/source/utf8.h
|
||||
)
|
||||
target_include_directories(utfcpp SYSTEM PUBLIC "lib/utfcpp-2.3.5/source")
|
||||
target_compile_options(utfcpp PRIVATE ${disableWarningsFlags})
|
||||
set_target_properties(utfcpp PROPERTIES FOLDER lib)
|
||||
|
||||
# ... utf8proc
|
||||
add_library(utf8proc
|
||||
lib/utf8proc-2a2f97e1/utf8proc.c
|
||||
lib/utf8proc-2a2f97e1/utf8proc.h
|
||||
)
|
||||
target_include_directories(utf8proc SYSTEM PUBLIC "lib/utf8proc-2a2f97e1")
|
||||
target_compile_options(utf8proc PRIVATE ${disableWarningsFlags})
|
||||
set_target_properties(utf8proc PROPERTIES FOLDER lib)
|
||||
|
||||
# Define Rhubarb libraries
|
||||
|
||||
# ... rhubarb-animation
|
||||
|
|
@ -401,6 +424,8 @@ target_include_directories(rhubarb-tools PUBLIC "src/tools")
|
|||
target_link_libraries(rhubarb-tools
|
||||
cppFormat
|
||||
whereami
|
||||
utfcpp
|
||||
utf8proc
|
||||
)
|
||||
|
||||
# Define Rhubarb executable
|
||||
|
|
|
|||
54
LICENSE.md
54
LICENSE.md
|
|
@ -162,6 +162,60 @@ The [TCLAP](http://tclap.sourceforge.net/) library is released under the **MIT L
|
|||
>
|
||||
> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
### `[utfcpp]` UTF8-CPP
|
||||
|
||||
The [UTF8-CPP](https://github.com/nemtrif/utfcpp) library is released under the **Boost Software License**.
|
||||
|
||||
> Copyright 2006 Nemanja Trifunovic
|
||||
>
|
||||
>Permission is hereby granted, free of charge, to any person or organization obtaining a copy of the software and accompanying documentation covered by this license (the "Software") to use, reproduce, display, distribute, execute, and transmit the Software, and to prepare derivative works of the Software, and to permit third-parties to whom the Software is furnished to do so, all subject to the following:
|
||||
>
|
||||
> The copyright notices in the Software and this entire statement, including the above license grant, this restriction and the following disclaimer, must be included in all copies of the Software, in whole or in part, and all derivative works of the Software, unless such copies or derivative works are solely in the form of machine-executable object code generated by a source language processor.
|
||||
>
|
||||
> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
### `[utf8proc]` utf8proc
|
||||
|
||||
The [utf8proc](https://github.com/JuliaLang/utf8proc) library is released under the **MIT License (MIT)**, while some of its data is released under the **UNICODE License**.
|
||||
|
||||
> #### utf8proc license
|
||||
>
|
||||
> **utf8proc** is a software package originally developed by Jan Behrens and the rest of the Public Software Group, who deserve nearly all of the credit for this library, that is now maintained by the Julia-language developers. Like the original utf8proc, whose copyright and license statements are reproduced below, all new work on the utf8proc library is licensed under the [MIT "expat" license](http://opensource.org/licenses/MIT):
|
||||
>
|
||||
> *Copyright © 2014-2015 by Steven G. Johnson, Jiahao Chen, Tony Kelman, Jonas Fonseca, and other contributors listed in the git history.*
|
||||
>
|
||||
> Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
>
|
||||
> The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
>
|
||||
> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
>
|
||||
> #### Original utf8proc license
|
||||
>
|
||||
> *Copyright (c) 2009, 2013 Public Software Group e. V., Berlin, Germany*
|
||||
>
|
||||
> Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
>
|
||||
> The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
>
|
||||
> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
>
|
||||
> #### Unicode data license
|
||||
>
|
||||
> This software distribution contains derived data from a modified version of the Unicode data files. The following license applies to that data:
|
||||
>
|
||||
> **COPYRIGHT AND PERMISSION NOTICE**
|
||||
>
|
||||
> *Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed under the Terms of Use in http://www.unicode.org/copyright.html.*
|
||||
>
|
||||
> Permission is hereby granted, free of charge, to any person obtaining a copy of the Unicode data files and any associated documentation (the "Data Files") or Unicode software and any associated documentation (the "Software") to deal in the Data Files or Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, and/or sell copies of the Data Files or Software, and to permit persons to whom the Data Files or Software are furnished to do so, provided that (a) the above copyright notice(s) and this permission notice appear with all copies of the Data Files or Software, (b) both the above copyright notice(s) and this permission notice appear in associated documentation, and (c) there is clear notice in each modified Data File or in the Software as well as in the documentation associated with the Data File(s) or Software that the data or software has been modified.
|
||||
>
|
||||
> THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA FILES OR SOFTWARE.
|
||||
>
|
||||
> Except as contained in this notice, the name of a copyright holder shall not be used in advertising or otherwise to promote the sale, use or other dealings in these Data Files or Software without prior written authorization of the copyright holder.
|
||||
>
|
||||
> Unicode and the Unicode logo are trademarks of Unicode, Inc., and may be registered in some jurisdictions. All other trademarks and registered trademarks mentioned herein are the property of their respective owners.
|
||||
|
||||
### `[webrtc]` WebRTC
|
||||
|
||||
The [WebRTC](https://chromium.googlesource.com/external/webrtc) library is released under the **3-clause BSD License**.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,7 @@
|
|||
/*
|
||||
|
||||
This library is header-only.
|
||||
Due to limitations in CMake, I added this dummy code file.
|
||||
For details, see http://cmake.3232098.n2.nabble.com/Cannot-set-FOLDER-property-to-an-interface-header-only-target-td7592375.html
|
||||
|
||||
*/
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
*.tar.gz
|
||||
*.exe
|
||||
*.dll
|
||||
*.do
|
||||
*.o
|
||||
*.so*
|
||||
*.a
|
||||
*.dll
|
||||
*.dylib
|
||||
*.dSYM
|
||||
*.out
|
||||
*.new
|
||||
data/*.txt
|
||||
data/*.ttf
|
||||
data/*.sfd
|
||||
/docs/
|
||||
bench/bench
|
||||
bench/icu
|
||||
bench/unistring
|
||||
test/normtest
|
||||
test/graphemetest
|
||||
test/printproperty
|
||||
test/charwidth
|
||||
test/valid
|
||||
test/iterate
|
||||
test/case
|
||||
test/custom
|
||||
/tmp/
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
language: c
|
||||
compiler:
|
||||
- gcc
|
||||
- clang
|
||||
notifications:
|
||||
email: false
|
||||
before_install:
|
||||
- sudo add-apt-repository ppa:staticfloat/julia-deps -y
|
||||
- sudo add-apt-repository ppa:staticfloat/juliareleases -y
|
||||
- sudo apt-get update -qq -y
|
||||
- sudo apt-get install libpcre3-dev julia fontforge -y
|
||||
script:
|
||||
- make manifest && diff MANIFEST.new MANIFEST
|
||||
- make check
|
||||
- make data && diff data/utf8proc_data.c.new utf8proc_data.c
|
||||
- make clean && git status --ignored --porcelain && test -z "$(git status --ignored --porcelain)"
|
||||
- (mkdir build_static && cd build_static && cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON && make)
|
||||
- (mkdir build_shared && cd build_shared && cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DBUILD_SHARED_LIBS=ON && make)
|
||||
env:
|
||||
# use JuliaLang caching (https://github.com/staticfloat/cache.julialang.org)
|
||||
# so that Travis builds do not depend on anyone's flaky servers but our own
|
||||
- URLCACHE=https://cache.julialang.org/ CFLAGS="-O2 -Werror -Wmissing-prototypes"
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
cmake_minimum_required (VERSION 2.8)
|
||||
|
||||
include (utils.cmake)
|
||||
|
||||
disallow_intree_builds()
|
||||
|
||||
project (utf8proc C)
|
||||
|
||||
# This is the ABI version number, which may differ from the
|
||||
# API version number (defined in utf8proc.h).
|
||||
# Be sure to also update these in Makefile and MANIFEST!
|
||||
set(SO_MAJOR 2)
|
||||
set(SO_MINOR 1)
|
||||
set(SO_PATCH 0)
|
||||
|
||||
add_definitions (
|
||||
-DUTF8PROC_EXPORTS
|
||||
)
|
||||
|
||||
if (NOT MSVC)
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2 -std=c99 -pedantic -Wall")
|
||||
endif ()
|
||||
|
||||
add_library (utf8proc
|
||||
utf8proc.c
|
||||
utf8proc.h
|
||||
)
|
||||
|
||||
set_target_properties (utf8proc PROPERTIES
|
||||
POSITION_INDEPENDENT_CODE ON
|
||||
VERSION "${SO_MAJOR}.${SO_MINOR}.${SO_PATCH}"
|
||||
SOVERSION ${SO_MAJOR}
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,93 @@
|
|||
## utf8proc license ##
|
||||
|
||||
**utf8proc** is a software package originally developed
|
||||
by Jan Behrens and the rest of the Public Software Group, who
|
||||
deserve nearly all of the credit for this library, that is now maintained by the Julia-language developers. Like the original utf8proc,
|
||||
whose copyright and license statements are reproduced below, all new
|
||||
work on the utf8proc library is licensed under the [MIT "expat"
|
||||
license](http://opensource.org/licenses/MIT):
|
||||
|
||||
*Copyright © 2014-2015 by Steven G. Johnson, Jiahao Chen, Tony Kelman, Jonas Fonseca, and other contributors listed in the git history.*
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a
|
||||
copy of this software and associated documentation files (the "Software"),
|
||||
to deal in the Software without restriction, including without limitation
|
||||
the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
and/or sell copies of the Software, and to permit persons to whom the
|
||||
Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
|
||||
## Original utf8proc license ##
|
||||
|
||||
*Copyright (c) 2009, 2013 Public Software Group e. V., Berlin, Germany*
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a
|
||||
copy of this software and associated documentation files (the "Software"),
|
||||
to deal in the Software without restriction, including without limitation
|
||||
the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
and/or sell copies of the Software, and to permit persons to whom the
|
||||
Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
|
||||
## Unicode data license ##
|
||||
|
||||
This software distribution contains derived data from a modified version of
|
||||
the Unicode data files. The following license applies to that data:
|
||||
|
||||
**COPYRIGHT AND PERMISSION NOTICE**
|
||||
|
||||
*Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed
|
||||
under the Terms of Use in http://www.unicode.org/copyright.html.*
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a
|
||||
copy of the Unicode data files and any associated documentation (the "Data
|
||||
Files") or Unicode software and any associated documentation (the
|
||||
"Software") to deal in the Data Files or Software without restriction,
|
||||
including without limitation the rights to use, copy, modify, merge,
|
||||
publish, distribute, and/or sell copies of the Data Files or Software, and
|
||||
to permit persons to whom the Data Files or Software are furnished to do
|
||||
so, provided that (a) the above copyright notice(s) and this permission
|
||||
notice appear with all copies of the Data Files or Software, (b) both the
|
||||
above copyright notice(s) and this permission notice appear in associated
|
||||
documentation, and (c) there is clear notice in each modified Data File or
|
||||
in the Software as well as in the documentation associated with the Data
|
||||
File(s) or Software that the data or software has been modified.
|
||||
|
||||
THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
|
||||
KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
|
||||
THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS
|
||||
INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR
|
||||
CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF
|
||||
USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
|
||||
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
PERFORMANCE OF THE DATA FILES OR SOFTWARE.
|
||||
|
||||
Except as contained in this notice, the name of a copyright holder shall
|
||||
not be used in advertising or otherwise to promote the sale, use or other
|
||||
dealings in these Data Files or Software without prior written
|
||||
authorization of the copyright holder.
|
||||
|
||||
Unicode and the Unicode logo are trademarks of Unicode, Inc., and may be
|
||||
registered in some jurisdictions. All other trademarks and registered
|
||||
trademarks mentioned herein are the property of their respective owners.
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
include/
|
||||
include/utf8proc.h
|
||||
lib/
|
||||
lib/libutf8proc.a
|
||||
lib/libutf8proc.so -> libutf8proc.so.2.1.0
|
||||
lib/libutf8proc.so.2 -> libutf8proc.so.2.1.0
|
||||
lib/libutf8proc.so.2.1.0
|
||||
|
|
@ -0,0 +1,149 @@
|
|||
# libutf8proc Makefile
|
||||
|
||||
# programs
|
||||
AR?=ar
|
||||
CC?=gcc
|
||||
INSTALL=install
|
||||
FIND=find
|
||||
|
||||
# compiler settings
|
||||
CFLAGS ?= -O2
|
||||
PICFLAG = -fPIC
|
||||
C99FLAG = -std=c99
|
||||
WCFLAGS = -Wall -pedantic
|
||||
UCFLAGS = $(CFLAGS) $(PICFLAG) $(C99FLAG) $(WCFLAGS) -DUTF8PROC_EXPORTS
|
||||
|
||||
# shared-library version MAJOR.MINOR.PATCH ... this may be *different*
|
||||
# from the utf8proc version number because it indicates ABI compatibility,
|
||||
# not API compatibility: MAJOR should be incremented whenever *binary*
|
||||
# compatibility is broken, even if the API is backward-compatible.
|
||||
# The API version number is defined in utf8proc.h.
|
||||
# Be sure to also update these ABI versions in MANIFEST and CMakeLists.txt!
|
||||
MAJOR=2
|
||||
MINOR=1
|
||||
PATCH=0
|
||||
|
||||
OS := $(shell uname)
|
||||
ifeq ($(OS),Darwin) # MacOS X
|
||||
SHLIB_EXT = dylib
|
||||
SHLIB_VERS_EXT = $(MAJOR).dylib
|
||||
else # GNU/Linux, at least (Windows should probably use cmake)
|
||||
SHLIB_EXT = so
|
||||
SHLIB_VERS_EXT = so.$(MAJOR).$(MINOR).$(PATCH)
|
||||
endif
|
||||
|
||||
# installation directories (for 'make install')
|
||||
prefix=/usr/local
|
||||
libdir=$(prefix)/lib
|
||||
includedir=$(prefix)/include
|
||||
|
||||
# meta targets
|
||||
|
||||
.PHONY: all clean data update manifest install
|
||||
|
||||
all: libutf8proc.a libutf8proc.$(SHLIB_EXT)
|
||||
|
||||
clean:
|
||||
rm -f utf8proc.o libutf8proc.a libutf8proc.$(SHLIB_VERS_EXT) libutf8proc.$(SHLIB_EXT)
|
||||
ifneq ($(OS),Darwin)
|
||||
rm -f libutf8proc.so.$(MAJOR)
|
||||
endif
|
||||
rm -f test/tests.o test/normtest test/graphemetest test/printproperty test/charwidth test/valid test/iterate test/case test/custom
|
||||
rm -rf MANIFEST.new tmp
|
||||
$(MAKE) -C bench clean
|
||||
$(MAKE) -C data clean
|
||||
|
||||
data: data/utf8proc_data.c.new
|
||||
|
||||
update: data/utf8proc_data.c.new
|
||||
cp -f data/utf8proc_data.c.new utf8proc_data.c
|
||||
|
||||
manifest: MANIFEST.new
|
||||
|
||||
# real targets
|
||||
|
||||
data/utf8proc_data.c.new: libutf8proc.$(SHLIB_EXT) data/data_generator.rb data/charwidths.jl
|
||||
$(MAKE) -C data utf8proc_data.c.new
|
||||
|
||||
utf8proc.o: utf8proc.h utf8proc.c utf8proc_data.c
|
||||
$(CC) $(UCFLAGS) -c -o utf8proc.o utf8proc.c
|
||||
|
||||
libutf8proc.a: utf8proc.o
|
||||
rm -f libutf8proc.a
|
||||
$(AR) rs libutf8proc.a utf8proc.o
|
||||
|
||||
libutf8proc.so.$(MAJOR).$(MINOR).$(PATCH): utf8proc.o
|
||||
$(CC) $(LDFLAGS) -shared -o $@ -Wl,-soname -Wl,libutf8proc.so.$(MAJOR) utf8proc.o
|
||||
chmod a-x $@
|
||||
|
||||
libutf8proc.so: libutf8proc.so.$(MAJOR).$(MINOR).$(PATCH)
|
||||
ln -f -s libutf8proc.so.$(MAJOR).$(MINOR).$(PATCH) $@
|
||||
ln -f -s libutf8proc.so.$(MAJOR).$(MINOR).$(PATCH) $@.$(MAJOR)
|
||||
|
||||
libutf8proc.$(MAJOR).dylib: utf8proc.o
|
||||
$(CC) -dynamiclib -o $@ $^ -install_name $(libdir)/$@ -Wl,-compatibility_version -Wl,$(MAJOR) -Wl,-current_version -Wl,$(MAJOR).$(MINOR).$(PATCH)
|
||||
|
||||
libutf8proc.dylib: libutf8proc.$(MAJOR).dylib
|
||||
ln -f -s libutf8proc.$(MAJOR).dylib $@
|
||||
|
||||
install: libutf8proc.a libutf8proc.$(SHLIB_EXT) libutf8proc.$(SHLIB_VERS_EXT)
|
||||
mkdir -m 755 -p $(DESTDIR)$(includedir)
|
||||
$(INSTALL) -m 644 utf8proc.h $(DESTDIR)$(includedir)
|
||||
mkdir -m 755 -p $(DESTDIR)$(libdir)
|
||||
$(INSTALL) -m 644 libutf8proc.a $(DESTDIR)$(libdir)
|
||||
$(INSTALL) -m 755 libutf8proc.$(SHLIB_VERS_EXT) $(DESTDIR)$(libdir)
|
||||
ln -f -s libutf8proc.$(SHLIB_VERS_EXT) $(DESTDIR)$(libdir)/libutf8proc.$(SHLIB_EXT)
|
||||
ifneq ($(OS),Darwin)
|
||||
ln -f -s libutf8proc.$(SHLIB_VERS_EXT) $(DESTDIR)$(libdir)/libutf8proc.so.$(MAJOR)
|
||||
endif
|
||||
|
||||
MANIFEST.new:
|
||||
rm -rf tmp
|
||||
$(MAKE) install prefix=/usr DESTDIR=$(PWD)/tmp
|
||||
$(FIND) tmp/usr -mindepth 1 -type l -printf "%P -> %l\n" -or -type f -printf "%P\n" -or -type d -printf "%P/\n" | LC_ALL=C sort > $@
|
||||
rm -rf tmp
|
||||
|
||||
# Test programs
|
||||
|
||||
data/NormalizationTest.txt:
|
||||
$(MAKE) -C data NormalizationTest.txt
|
||||
|
||||
data/GraphemeBreakTest.txt:
|
||||
$(MAKE) -C data GraphemeBreakTest.txt
|
||||
|
||||
test/tests.o: test/tests.c test/tests.h utf8proc.h
|
||||
$(CC) $(UCFLAGS) -c -o test/tests.o test/tests.c
|
||||
|
||||
test/normtest: test/normtest.c test/tests.o utf8proc.o utf8proc.h test/tests.h
|
||||
$(CC) $(UCFLAGS) test/normtest.c test/tests.o utf8proc.o -o $@
|
||||
|
||||
test/graphemetest: test/graphemetest.c test/tests.o utf8proc.o utf8proc.h test/tests.h
|
||||
$(CC) $(UCFLAGS) test/graphemetest.c test/tests.o utf8proc.o -o $@
|
||||
|
||||
test/printproperty: test/printproperty.c test/tests.o utf8proc.o utf8proc.h test/tests.h
|
||||
$(CC) $(UCFLAGS) test/printproperty.c test/tests.o utf8proc.o -o $@
|
||||
|
||||
test/charwidth: test/charwidth.c test/tests.o utf8proc.o utf8proc.h test/tests.h
|
||||
$(CC) $(UCFLAGS) test/charwidth.c test/tests.o utf8proc.o -o $@
|
||||
|
||||
test/valid: test/valid.c test/tests.o utf8proc.o utf8proc.h test/tests.h
|
||||
$(CC) $(UCFLAGS) test/valid.c test/tests.o utf8proc.o -o $@
|
||||
|
||||
test/iterate: test/iterate.c test/tests.o utf8proc.o utf8proc.h test/tests.h
|
||||
$(CC) $(UCFLAGS) test/iterate.c test/tests.o utf8proc.o -o $@
|
||||
|
||||
test/case: test/case.c test/tests.o utf8proc.o utf8proc.h test/tests.h
|
||||
$(CC) $(UCFLAGS) test/case.c test/tests.o utf8proc.o -o $@
|
||||
|
||||
test/custom: test/custom.c test/tests.o utf8proc.o utf8proc.h test/tests.h
|
||||
$(CC) $(UCFLAGS) test/custom.c test/tests.o utf8proc.o -o $@
|
||||
|
||||
check: test/normtest data/NormalizationTest.txt test/graphemetest data/GraphemeBreakTest.txt test/printproperty test/case test/custom test/charwidth test/valid test/iterate bench/bench.c bench/util.c bench/util.h utf8proc.o
|
||||
$(MAKE) -C bench
|
||||
test/normtest data/NormalizationTest.txt
|
||||
test/graphemetest data/GraphemeBreakTest.txt
|
||||
test/charwidth
|
||||
test/valid
|
||||
test/iterate
|
||||
test/case
|
||||
test/custom
|
||||
|
|
@ -0,0 +1,303 @@
|
|||
# utf8proc release history #
|
||||
|
||||
## Version 2.1 ##
|
||||
|
||||
2016-12-26:
|
||||
|
||||
- New functions `utf8proc_map_custom` and `utf8proc_decompose_custom`
|
||||
to allow user-supplied transformations of codepoints, in conjunction
|
||||
with other transformations ([#89]).
|
||||
|
||||
- New function `utf8proc_normalize_utf32` to apply normalizations
|
||||
directly to UTF-32 data (not just UTF-8) ([#88]).
|
||||
|
||||
- Fixed stack overflow that could occur due to incorrect definition
|
||||
of `UINT16_MAX` with some compilers ([#84]).
|
||||
|
||||
- Fixed conflict with `stdbool.h` in Visual Studio ([#90]).
|
||||
|
||||
- Updated font metrics to use Unifont 9.0.04.
|
||||
|
||||
## Version 2.0.2 ##
|
||||
|
||||
2016-07-27:
|
||||
|
||||
- Move `-Wmissing-prototypes` warning flag from `Makefile` to `.travis.yml`
|
||||
since MSVC does not understand this flag and it is occasionally useful to
|
||||
build using MSVC through the `Makefile` ([#79]).
|
||||
|
||||
- Use a different variable name for a nested loop in `bench/bench.c`, and
|
||||
declare it in a C89 way rather than inside the `for` to avoid "error:
|
||||
'for' loop initial declarations are only allowed in C99 mode" ([#80]).
|
||||
|
||||
## Version 2.0.1 ##
|
||||
|
||||
2016-07-13:
|
||||
|
||||
- Bug fix in `utf8proc_grapheme_break_stateful` ([#77]).
|
||||
|
||||
- Tests now use versioned Unicode files, so they will no longer
|
||||
break when a new version of Unicode is released ([#78]).
|
||||
|
||||
## Version 2.0 ##
|
||||
|
||||
2016-07-13:
|
||||
|
||||
- Updated for Unicode 9.0 ([#70]).
|
||||
|
||||
- New `utf8proc_grapheme_break_stateful` to handle the complicated
|
||||
grapheme-breaking rules in Unicode 9. The old `utf8proc_grapheme_break`
|
||||
is still provided, but may incorrectly identify grapheme breaks
|
||||
in some Unicode-9 sequences.
|
||||
|
||||
- Smaller Unicode tables ([#62], [#68]). This required changes
|
||||
in the `utf8proc_property_t` structure, which breaks backward
|
||||
compatibility if you access this `struct` directly. The
|
||||
functions in the API remain backward-compatible, however.
|
||||
|
||||
- Buffer overrun fix ([#66]).
|
||||
|
||||
## Version 1.3.1 ##
|
||||
|
||||
2015-11-02:
|
||||
|
||||
- Do not export symbol for internal function `unsafe_encode_char()` ([#55]).
|
||||
|
||||
- Install relative symbolic links for shared libraries ([#58]).
|
||||
|
||||
- Enable and fix compiler warnings ([#55], [#58]).
|
||||
|
||||
- Add missing files to `make clean` ([#58]).
|
||||
|
||||
## Version 1.3 ##
|
||||
|
||||
2015-07-06:
|
||||
|
||||
- Updated for Unicode 8.0 ([#45]).
|
||||
|
||||
- New `utf8proc_tolower` and `utf8proc_toupper` functions, portable
|
||||
replacements for `towlower` and `towupper` in the C library ([#40]).
|
||||
|
||||
- Don't treat Unicode "non-characters" as invalid, and improved
|
||||
validity checking in general ([#35]).
|
||||
|
||||
- Prefix all typedefs with `utf8proc_`, e.g. `utf8proc_int32_t`,
|
||||
to avoid collisions with other libraries ([#32]).
|
||||
|
||||
- Rename `DLLEXPORT` to `UTF8PROC_DLLEXPORT` to prevent collisions.
|
||||
|
||||
- Fix build breakage in the benchmark routines.
|
||||
|
||||
- More fine-grained Makefile variables (`PICFLAG` etcetera), so that
|
||||
compilation flags can be selectively overridden, and in particular
|
||||
so that `CFLAGS` can be changed without accidentally eliminating
|
||||
necessary flags like `-fPIC` and `-std=c99` ([#43]).
|
||||
|
||||
- Updated character-width tables based on Unifont 8.0.01 ([#51]) and
|
||||
the Unicode 8 character categories ([#47]).
|
||||
|
||||
## Version 1.2 ##
|
||||
|
||||
2015-03-28:
|
||||
|
||||
- Updated for Unicode 7.0 ([#6]).
|
||||
|
||||
- New function `utf8proc_grapheme_break(c1,c2)` that returns whether
|
||||
there is a grapheme break between `c1` and `c2` ([#20]).
|
||||
|
||||
- New function `utf8proc_charwidth(c)` that returns the number of
|
||||
column-positions that should be required for `c`; essentially a
|
||||
portable replacment for `wcwidth(c)` ([#27]).
|
||||
|
||||
- New function `utf8proc_category(c)` that returns the Unicode
|
||||
category of `c` (as one of the constants `UTF8PROC_CATEGORY_xx`).
|
||||
Also, a function `utf8proc_category_string(c)` that returns the Unicode
|
||||
category of `c` as a two-character string.
|
||||
|
||||
- `cmake` script `CMakeLists.txt`, in addition to `Makefile`, for
|
||||
easier compilation on Windows ([#28]).
|
||||
|
||||
- Various `Makefile` improvements: a `make check` target to perform
|
||||
tests ([#13]), `make install`, a rule to automate updating the Unicode
|
||||
tables, etcetera.
|
||||
|
||||
- The shared library is now versioned (e.g. has a soname on GNU/Linux) ([#24]).
|
||||
|
||||
- C++/MSVC compatibility ([#17]).
|
||||
|
||||
- Most `#defined` constants are now `enums` ([#29]).
|
||||
|
||||
- New preprocessor constants `UTF8PROC_VERSION_MAJOR`,
|
||||
`UTF8PROC_VERSION_MINOR`, and `UTF8PROC_VERSION_PATCH` for compile-time
|
||||
detection of the API version.
|
||||
|
||||
- Doxygen-formatted documentation ([#29]).
|
||||
|
||||
- The Ruby and PostgreSQL plugins have been removed due to lack of testing ([#22]).
|
||||
|
||||
## Version 1.1.6 ##
|
||||
|
||||
2013-11-27:
|
||||
|
||||
- PostgreSQL 9.2 and 9.3 compatibility (lowercase `c` language name)
|
||||
|
||||
## Version 1.1.5 ##
|
||||
|
||||
2009-08-20:
|
||||
|
||||
- Use `RSTRING_PTR()` and `RSTRING_LEN()` instead of `RSTRING()->ptr` and
|
||||
`RSTRING()->len` for ruby1.9 compatibility (and `#define` them, if not
|
||||
existent)
|
||||
|
||||
2009-10-02:
|
||||
|
||||
- Patches for compatibility with Microsoft Visual Studio
|
||||
|
||||
2009-10-08:
|
||||
|
||||
- Fixes to make utf8proc usable in C++ programs
|
||||
|
||||
2009-10-16:
|
||||
|
||||
## Version 1.1.4 ##
|
||||
|
||||
2009-06-14:
|
||||
|
||||
- replaced C++ style comments for compatibility reasons
|
||||
- added typecasts to suppress compiler warnings
|
||||
- removed redundant source files for ruby-gemfile generation
|
||||
|
||||
2009-08-19:
|
||||
|
||||
- Changed copyright notice for Public Software Group e. V.
|
||||
- Minor changes in the `README` file
|
||||
|
||||
## Version 1.1.3 ##
|
||||
|
||||
2008-10-04:
|
||||
|
||||
- Added a function `utf8proc_version` returning a string containing the version
|
||||
number of the library.
|
||||
- Included a target `libutf8proc.dylib` for MacOSX.
|
||||
|
||||
2009-05-01:
|
||||
- PostgreSQL 8.3 compatibility (use of `SET_VARSIZE` macro)
|
||||
|
||||
## Version 1.1.2 ##
|
||||
|
||||
2007-07-25:
|
||||
|
||||
- Fixed a serious bug in the data file generator, which caused characters
|
||||
being treated incorrectly, when stripping default ignorable characters or
|
||||
calculating grapheme cluster boundaries.
|
||||
|
||||
## Version 1.1.1 ##
|
||||
|
||||
2007-06-25:
|
||||
|
||||
- Added a new PostgreSQL function `unistrip`, which behaves like `unifold`,
|
||||
but also removes all character marks (e.g. accents).
|
||||
|
||||
2007-07-22:
|
||||
|
||||
- Changed license from BSD to MIT style.
|
||||
- Added a new function `utf8proc_codepoint_valid` to the C library.
|
||||
- Changed compiler flags in `Makefile` from `-g -O0` to `-O2`
|
||||
- The ruby script, which was used to build the `utf8proc_data.c` file, is now
|
||||
included in the distribution.
|
||||
|
||||
## Version 1.0.3 ##
|
||||
|
||||
2007-03-16:
|
||||
|
||||
- Fixed a bug in the ruby library, which caused an error, when splitting an
|
||||
empty string at grapheme cluster boundaries (method `String#utf8chars`).
|
||||
|
||||
## Version 1.0.2 ##
|
||||
|
||||
2006-09-21:
|
||||
|
||||
- included a check in `Integer#utf8`, which raises an exception, if the given
|
||||
code-point is invalid because of being too high (this was missing yet)
|
||||
|
||||
2006-12-26:
|
||||
|
||||
- added support for PostgreSQL version 8.2
|
||||
|
||||
## Version 1.0.1 ##
|
||||
|
||||
2006-09-20:
|
||||
|
||||
- included a gem file for the ruby version of the library
|
||||
|
||||
Release of version 1.0.1
|
||||
|
||||
## Version 1.0 ##
|
||||
|
||||
2006-09-17:
|
||||
|
||||
- added the `LUMP` option, which lumps certain characters together (see `lump.md`) (also used for the PostgreSQL `unifold` function)
|
||||
- added the `STRIPMARK` option, which strips marking characters (or marks of composed characters)
|
||||
- deprecated ruby method `String#char_ary` in favour of `String#utf8chars`
|
||||
|
||||
## Version 0.3 ##
|
||||
|
||||
2006-07-18:
|
||||
|
||||
- changed normalization from NFC to NFKC for postgresql unifold function
|
||||
|
||||
2006-08-04:
|
||||
|
||||
- added support to mark the beginning of a grapheme cluster with 0xFF (option: `CHARBOUND`)
|
||||
- added the ruby method `String#chars`, which is returning an array of UTF-8 encoded grapheme clusters
|
||||
- added `NLF2LF` transformation in postgresql `unifold` function
|
||||
- added the `DECOMPOSE` option, if you neither use `COMPOSE` or `DECOMPOSE`, no normalization will be performed (different from previous versions)
|
||||
- using integer constants rather than C-strings for character properties
|
||||
- fixed (hopefully) a problem with the ruby library on Mac OS X, which occurred when compiler optimization was switched on
|
||||
|
||||
## Version 0.2 ##
|
||||
|
||||
2006-06-05:
|
||||
|
||||
- changed behaviour of PostgreSQL function to return NULL in case of invalid input, rather than raising an exceptional condition
|
||||
- improved efficiency of PostgreSQL function (no transformation to C string is done)
|
||||
|
||||
2006-06-20:
|
||||
|
||||
- added -fpic compiler flag in Makefile
|
||||
- fixed bug in the C code for the ruby library (usage of non-existent function)
|
||||
|
||||
## Version 0.1 ##
|
||||
|
||||
2006-06-02: initial release of version 0.1
|
||||
|
||||
[#6]: https://github.com/JuliaLang/utf8proc/issues/6
|
||||
[#13]: https://github.com/JuliaLang/utf8proc/issues/13
|
||||
[#17]: https://github.com/JuliaLang/utf8proc/issues/17
|
||||
[#20]: https://github.com/JuliaLang/utf8proc/issues/20
|
||||
[#22]: https://github.com/JuliaLang/utf8proc/issues/22
|
||||
[#24]: https://github.com/JuliaLang/utf8proc/issues/24
|
||||
[#27]: https://github.com/JuliaLang/utf8proc/issues/27
|
||||
[#28]: https://github.com/JuliaLang/utf8proc/issues/28
|
||||
[#29]: https://github.com/JuliaLang/utf8proc/issues/29
|
||||
[#32]: https://github.com/JuliaLang/utf8proc/issues/32
|
||||
[#35]: https://github.com/JuliaLang/utf8proc/issues/35
|
||||
[#40]: https://github.com/JuliaLang/utf8proc/issues/40
|
||||
[#43]: https://github.com/JuliaLang/utf8proc/issues/43
|
||||
[#45]: https://github.com/JuliaLang/utf8proc/issues/45
|
||||
[#47]: https://github.com/JuliaLang/utf8proc/issues/47
|
||||
[#51]: https://github.com/JuliaLang/utf8proc/issues/51
|
||||
[#55]: https://github.com/JuliaLang/utf8proc/issues/55
|
||||
[#58]: https://github.com/JuliaLang/utf8proc/issues/58
|
||||
[#62]: https://github.com/JuliaLang/utf8proc/issues/62
|
||||
[#66]: https://github.com/JuliaLang/utf8proc/issues/66
|
||||
[#68]: https://github.com/JuliaLang/utf8proc/issues/68
|
||||
[#70]: https://github.com/JuliaLang/utf8proc/issues/70
|
||||
[#77]: https://github.com/JuliaLang/utf8proc/issues/77
|
||||
[#78]: https://github.com/JuliaLang/utf8proc/issues/78
|
||||
[#79]: https://github.com/JuliaLang/utf8proc/issues/79
|
||||
[#80]: https://github.com/JuliaLang/utf8proc/issues/80
|
||||
[#84]: https://github.com/JuliaLang/utf8proc/pull/84
|
||||
[#88]: https://github.com/JuliaLang/utf8proc/pull/88
|
||||
[#89]: https://github.com/JuliaLang/utf8proc/pull/89
|
||||
[#90]: https://github.com/JuliaLang/utf8proc/issues/90
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
# utf8proc
|
||||
[](https://travis-ci.org/JuliaLang/utf8proc)
|
||||
[](https://ci.appveyor.com/project/tkelman/utf8proc/branch/master)
|
||||
|
||||
|
||||
[utf8proc](http://julialang.org/utf8proc/) is a small, clean C
|
||||
library that provides Unicode normalization, case-folding, and other
|
||||
operations for data in the [UTF-8
|
||||
encoding](http://en.wikipedia.org/wiki/UTF-8). It was [initially
|
||||
developed](http://www.public-software-group.org/utf8proc) by Jan
|
||||
Behrens and the rest of the [Public Software
|
||||
Group](http://www.public-software-group.org/), who deserve *nearly all
|
||||
of the credit* for this package. With the blessing of the Public
|
||||
Software Group, the [Julia developers](http://julialang.org/) have
|
||||
taken over development of utf8proc, since the original developers have
|
||||
moved to other projects.
|
||||
|
||||
(utf8proc is used for basic Unicode
|
||||
support in the [Julia language](http://julialang.org/), and the Julia
|
||||
developers became involved because they wanted to add Unicode 7 support and other features.)
|
||||
|
||||
(The original utf8proc package also includes Ruby and PostgreSQL plug-ins.
|
||||
We removed those from utf8proc in order to focus exclusively on the C
|
||||
library for the time being, but plan to add them back in or release them as separate packages.)
|
||||
|
||||
The utf8proc package is licensed under the
|
||||
free/open-source [MIT "expat"
|
||||
license](http://opensource.org/licenses/MIT) (plus certain Unicode
|
||||
data governed by the similarly permissive [Unicode data
|
||||
license](http://www.unicode.org/copyright.html#Exhibit1)); please see
|
||||
the included `LICENSE.md` file for more detailed information.
|
||||
|
||||
## Quick Start
|
||||
|
||||
For compilation of the C library run `make`.
|
||||
|
||||
## General Information
|
||||
|
||||
The C library is found in this directory after successful compilation
|
||||
and is named `libutf8proc.a` (for the static library) and
|
||||
`libutf8proc.so` (for the dynamic library).
|
||||
|
||||
The Unicode version supported is 9.0.0.
|
||||
|
||||
For Unicode normalizations, the following options are used:
|
||||
|
||||
* Normalization Form C: `STABLE`, `COMPOSE`
|
||||
* Normalization Form D: `STABLE`, `DECOMPOSE`
|
||||
* Normalization Form KC: `STABLE`, `COMPOSE`, `COMPAT`
|
||||
* Normalization Form KD: `STABLE`, `DECOMPOSE`, `COMPAT`
|
||||
|
||||
## C Library
|
||||
|
||||
The documentation for the C library is found in the `utf8proc.h` header file.
|
||||
`utf8proc_map` is function you will most likely be using for mapping UTF-8
|
||||
strings, unless you want to allocate memory yourself.
|
||||
|
||||
## To Do
|
||||
|
||||
See the Github [issues list](https://github.com/JuliaLang/utf8proc/issues).
|
||||
|
||||
## Contact
|
||||
|
||||
Bug reports, feature requests, and other queries can be filed at
|
||||
the [utf8proc issues page on Github](https://github.com/JuliaLang/utf8proc/issues).
|
||||
|
||||
## See also
|
||||
|
||||
An independent Lua translation of this library, [lua-mojibake](https://github.com/differentprogramming/lua-mojibake), is also available.
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
branches:
|
||||
only:
|
||||
- master
|
||||
- /release-.*/
|
||||
|
||||
notifications:
|
||||
- provider: Email
|
||||
on_build_success: false
|
||||
on_build_failure: false
|
||||
on_build_status_changed: false
|
||||
|
||||
build_script:
|
||||
- ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod `
|
||||
https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | `
|
||||
Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { `
|
||||
throw "There are newer queued builds for this pull request, failing early." }
|
||||
- mkdir msvc_static
|
||||
- cd msvc_static
|
||||
- cmake ..
|
||||
- cmake --build .
|
||||
- mkdir ..\msvc_shared
|
||||
- cd ..\msvc_shared
|
||||
- cmake .. -DBUILD_SHARED_LIBS=ON
|
||||
- cmake --build .
|
||||
- set PATH=C:\MinGW\bin;%PATH%
|
||||
- C:\MinGW\msys\1.0\bin\sh --login -c "
|
||||
cd /c/projects/utf8proc &&
|
||||
mkdir mingw_static &&
|
||||
cd mingw_static &&
|
||||
cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -G'MSYS Makefiles' &&
|
||||
make &&
|
||||
mkdir ../mingw_shared &&
|
||||
cd ../mingw_shared &&
|
||||
cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DBUILD_SHARED_LIBS=ON -G'MSYS Makefiles' &&
|
||||
make
|
||||
"
|
||||
|
||||
on_finish:
|
||||
# Uncomment the following line for interactive debugging, which
|
||||
# will print login data for a temporary remote session after the
|
||||
# build. This requires an RDP version 6 client, e.g., FreeRDP.
|
||||
#- ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
CURL=curl
|
||||
|
||||
CC = cc
|
||||
CFLAGS = -O2 -std=c99 -pedantic -Wall
|
||||
|
||||
all: bench
|
||||
|
||||
LIBUTF8PROC = ../utf8proc.o
|
||||
|
||||
bench: bench.o util.o $(LIBUTF8PROC)
|
||||
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ bench.o util.o $(LIBUTF8PROC)
|
||||
|
||||
DATAURL = https://raw.githubusercontent.com/duerst/eprun/master/benchmark
|
||||
DATAFILES = Deutsch_.txt Japanese_.txt Korean_.txt Vietnamese_.txt
|
||||
|
||||
$(DATAFILES):
|
||||
$(CURL) -O $(DATAURL)/$@
|
||||
|
||||
bench.out: $(DATAFILES) bench
|
||||
./bench -nfkc $(DATAFILES) > $@
|
||||
|
||||
# you may need make CPPFLAGS=... LDFLAGS=... to help it find ICU
|
||||
icu: icu.o util.o
|
||||
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ icu.o util.o -licuuc
|
||||
|
||||
icu.out: $(DATAFILES) icu
|
||||
./icu $(DATAFILES) > $@
|
||||
|
||||
unistring: unistring.o util.o
|
||||
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ unistring.o util.o -lunistring
|
||||
|
||||
unistring.out: $(DATAFILES) unistring
|
||||
./unistring $(DATAFILES) > $@
|
||||
|
||||
.c.o:
|
||||
$(CC) $(CPPFLAGS) -I.. $(CFLAGS) -c -o $@ $<
|
||||
|
||||
clean:
|
||||
rm -rf *.o *.txt bench *.out icu unistring
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "utf8proc.h"
|
||||
#include "util.h"
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
int i, j;
|
||||
int options = 0;
|
||||
|
||||
for (i = 1; i < argc; ++i) {
|
||||
if (!strcmp(argv[i], "-nfkc")) {
|
||||
options |= UTF8PROC_STABLE|UTF8PROC_COMPOSE|UTF8PROC_COMPAT;
|
||||
continue;
|
||||
}
|
||||
if (!strcmp(argv[i], "-nfkd")) {
|
||||
options |= UTF8PROC_STABLE|UTF8PROC_DECOMPOSE|UTF8PROC_COMPAT;
|
||||
continue;
|
||||
}
|
||||
if (!strcmp(argv[i], "-nfc")) {
|
||||
options |= UTF8PROC_STABLE|UTF8PROC_COMPOSE;
|
||||
continue;
|
||||
}
|
||||
if (!strcmp(argv[i], "-nfd")) {
|
||||
options |= UTF8PROC_STABLE|UTF8PROC_DECOMPOSE;
|
||||
continue;
|
||||
}
|
||||
if (!strcmp(argv[i], "-casefold")) {
|
||||
options |= UTF8PROC_CASEFOLD;
|
||||
continue;
|
||||
}
|
||||
if (argv[i][0] == '-') {
|
||||
fprintf(stderr, "unrecognized option: %s\n", argv[i]);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
size_t len;
|
||||
uint8_t *src = readfile(argv[i], &len);
|
||||
if (!src) {
|
||||
fprintf(stderr, "error reading %s\n", argv[i]);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
uint8_t *dest;
|
||||
mytime start = gettime();
|
||||
for (j = 0; j < 100; ++j) {
|
||||
utf8proc_map(src, len, &dest, options);
|
||||
free(dest);
|
||||
}
|
||||
printf("%s: %g\n", argv[i], elapsed(gettime(), start) / 100);
|
||||
free(src);
|
||||
}
|
||||
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
/* ICU4C */
|
||||
#include <unicode/utypes.h>
|
||||
#include <unicode/ustring.h>
|
||||
#include <unicode/ucnv.h>
|
||||
#include <unicode/unorm2.h>
|
||||
|
||||
#include "util.h"
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
int i;
|
||||
|
||||
UErrorCode err;
|
||||
UConverter *uc = ucnv_open("UTF8", &err);
|
||||
if (U_FAILURE(err)) return EXIT_FAILURE;
|
||||
|
||||
const UNormalizer2 *NFKC = unorm2_getNFKCInstance(&err);
|
||||
if (U_FAILURE(err)) return EXIT_FAILURE;
|
||||
|
||||
for (i = 1; i < argc; ++i) {
|
||||
if (argv[i][0] == '-') {
|
||||
fprintf(stderr, "unrecognized option: %s\n", argv[i]);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
size_t len;
|
||||
uint8_t *src = readfile(argv[i], &len);
|
||||
if (!src) {
|
||||
fprintf(stderr, "error reading %s\n", argv[i]);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
/* convert UTF8 data to ICU's UTF16 */
|
||||
UChar *usrc = (UChar*) malloc(2*len * sizeof(UChar));
|
||||
ucnv_toUChars(uc, usrc, 2*len, (char*) src, len, &err);
|
||||
if (U_FAILURE(err)) return EXIT_FAILURE;
|
||||
size_t ulen = u_strlen(usrc);
|
||||
|
||||
/* ICU's insane normalization API requires you to
|
||||
know the size of the destination buffer in advance,
|
||||
or alternatively to repeatly try normalizing and
|
||||
double the buffer size until it succeeds. Here, I just
|
||||
allocate a huge destination buffer to avoid the issue. */
|
||||
UChar *udest = (UChar*) malloc(10*ulen * sizeof(UChar));
|
||||
|
||||
mytime start = gettime();
|
||||
for (int i = 0; i < 100; ++i) {
|
||||
unorm2_normalize(NFKC, usrc, ulen, udest, 10*ulen, &err);
|
||||
if (U_FAILURE(err)) return EXIT_FAILURE;
|
||||
}
|
||||
printf("%s: %g\n", argv[i], elapsed(gettime(), start) / 100);
|
||||
free(udest);
|
||||
free(usrc);
|
||||
free(src);
|
||||
}
|
||||
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
/* comparitive benchmark of GNU libunistring */
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
/* libunistring */
|
||||
#include <unistr.h>
|
||||
#include <uninorm.h>
|
||||
|
||||
#include "util.h"
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
int i;
|
||||
uninorm_t nf = UNINORM_NFKC;
|
||||
|
||||
for (i = 1; i < argc; ++i) {
|
||||
if (!strcmp(argv[i], "-nfkc")) {
|
||||
nf = UNINORM_NFKC;
|
||||
continue;
|
||||
}
|
||||
if (!strcmp(argv[i], "-nfkd")) {
|
||||
nf = UNINORM_NFKD;
|
||||
continue;
|
||||
}
|
||||
if (!strcmp(argv[i], "-nfc")) {
|
||||
nf = UNINORM_NFC;
|
||||
continue;
|
||||
}
|
||||
if (!strcmp(argv[i], "-nfd")) {
|
||||
nf = UNINORM_NFD;
|
||||
continue;
|
||||
}
|
||||
if (argv[i][0] == '-') {
|
||||
fprintf(stderr, "unrecognized option: %s\n", argv[i]);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
size_t len;
|
||||
uint8_t *src = readfile(argv[i], &len);
|
||||
if (!src) {
|
||||
fprintf(stderr, "error reading %s\n", argv[i]);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
size_t destlen;
|
||||
uint8_t *dest;
|
||||
mytime start = gettime();
|
||||
for (int i = 0; i < 100; ++i) {
|
||||
dest = u8_normalize(nf, src, len, NULL, &destlen);
|
||||
if (!dest) return EXIT_FAILURE;
|
||||
free(dest);
|
||||
}
|
||||
printf("%s: %g\n", argv[i], elapsed(gettime(), start) / 100);
|
||||
free(src);
|
||||
}
|
||||
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
#include "util.h"
|
||||
|
||||
/* read file named FILENAME into an array of *len bytes,
|
||||
returning NULL on error */
|
||||
uint8_t *readfile(const char *filename, size_t *len)
|
||||
{
|
||||
*len = 0;
|
||||
struct stat st;
|
||||
if (0 != stat(filename, &st)) return NULL;
|
||||
*len = st.st_size;
|
||||
FILE *f = fopen(filename, "r");
|
||||
if (!f) return NULL;
|
||||
uint8_t *s = (uint8_t *) malloc(sizeof(uint8_t) * *len);
|
||||
if (!s) return NULL;
|
||||
if (fread(s, 1, *len, f) != *len) {
|
||||
free(s);
|
||||
s = NULL;
|
||||
}
|
||||
fclose(f);
|
||||
return s;
|
||||
}
|
||||
|
||||
mytime gettime(void) {
|
||||
mytime t;
|
||||
gettimeofday(&t, NULL);
|
||||
return t;
|
||||
}
|
||||
|
||||
/* time difference in seconds */
|
||||
double elapsed(mytime t1, mytime t0)
|
||||
{
|
||||
return (double)(t1.tv_sec - t0.tv_sec) +
|
||||
(double)(t1.tv_usec - t0.tv_usec) * 1.0E-6;
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
#ifndef UTIL_H
|
||||
#define UTIL_H 1
|
||||
|
||||
#include <inttypes.h>
|
||||
#include <sys/time.h>
|
||||
#include <time.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
uint8_t *readfile(const char *filename, size_t *len);
|
||||
|
||||
typedef struct timeval mytime;
|
||||
mytime gettime(void);
|
||||
double elapsed(mytime t1, mytime t0);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* UTIL_H */
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
# Unicode data generation rules. Except for the test data files, most
|
||||
# users will not use these Makefile rules, which are primarily to re-generate
|
||||
# unicode_data.c when we get a new Unicode version or charwidth data; they
|
||||
# require ruby, fontforge, and julia to be installed.
|
||||
|
||||
# programs
|
||||
CURL=curl
|
||||
RUBY=ruby
|
||||
PERL=perl
|
||||
MAKE=make
|
||||
JULIA=julia
|
||||
FONTFORGE=fontforge
|
||||
CURLFLAGS = --retry 5 --location
|
||||
|
||||
.PHONY: clean
|
||||
|
||||
.DELETE_ON_ERROR:
|
||||
|
||||
utf8proc_data.c.new: data_generator.rb UnicodeData.txt GraphemeBreakProperty.txt DerivedCoreProperties.txt CompositionExclusions.txt CaseFolding.txt CharWidths.txt
|
||||
$(RUBY) data_generator.rb < UnicodeData.txt > $@
|
||||
|
||||
# GNU Unifont version for font metric calculations:
|
||||
UNIFONT_VERSION=9.0.04
|
||||
|
||||
unifont.ttf:
|
||||
$(CURL) $(CURLFLAGS) -o $@ $(URLCACHE)https://mirrors.kernel.org/gnu/unifont/unifont-$(UNIFONT_VERSION)/unifont-$(UNIFONT_VERSION).ttf
|
||||
|
||||
unifont_upper.ttf:
|
||||
$(CURL) $(CURLFLAGS) -o $@ $(URLCACHE)https://mirrors.kernel.org/gnu/unifont/unifont-$(UNIFONT_VERSION)/unifont_upper-$(UNIFONT_VERSION).ttf
|
||||
|
||||
%.sfd: %.ttf
|
||||
$(FONTFORGE) -lang=ff -c "Open(\"$<\");Save(\"$@\");Quit(0);"
|
||||
|
||||
CharWidths.txt: charwidths.jl unifont.sfd unifont_upper.sfd EastAsianWidth.txt
|
||||
$(JULIA) charwidths.jl > $@
|
||||
|
||||
# Unicode data version
|
||||
UNICODE_VERSION=9.0.0
|
||||
|
||||
UnicodeData.txt:
|
||||
$(CURL) $(CURLFLAGS) -o $@ -O http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/UnicodeData.txt
|
||||
|
||||
EastAsianWidth.txt:
|
||||
$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/EastAsianWidth.txt
|
||||
|
||||
GraphemeBreakProperty.txt:
|
||||
$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/auxiliary/GraphemeBreakProperty.txt
|
||||
|
||||
DerivedCoreProperties.txt:
|
||||
$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/DerivedCoreProperties.txt
|
||||
|
||||
CompositionExclusions.txt:
|
||||
$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/CompositionExclusions.txt
|
||||
|
||||
CaseFolding.txt:
|
||||
$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/CaseFolding.txt
|
||||
|
||||
NormalizationTest.txt:
|
||||
$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/NormalizationTest.txt
|
||||
|
||||
GraphemeBreakTest.txt:
|
||||
$(CURL) $(CURLFLAGS) $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/auxiliary/GraphemeBreakTest.txt | $(PERL) -pe 's,÷,/,g;s,×,+,g' > $@
|
||||
|
||||
clean:
|
||||
rm -f UnicodeData.txt EastAsianWidth.txt GraphemeBreakProperty.txt DerivedCoreProperties.txt CompositionExclusions.txt CaseFolding.txt NormalizationTest.txt GraphemeBreakTest.txt CharWidths.txt unifont*.ttf unifont*.sfd
|
||||
rm -f utf8proc_data.c.new
|
||||
|
|
@ -0,0 +1,190 @@
|
|||
# Following work by @jiahao, we compute character widths using a combination of
|
||||
# * advance widths from GNU Unifont (advance width 512 = 1 en)
|
||||
# * UAX 11: East Asian Width
|
||||
# * a few exceptions as needed
|
||||
# Adapted from http://nbviewer.ipython.org/gist/jiahao/07e8b08bf6d8671e9734
|
||||
#
|
||||
# Requires Julia (obviously) and FontForge.
|
||||
|
||||
#############################################################################
|
||||
# Julia 0.3/0.4 compatibility (taken from Compat package)
|
||||
if VERSION < v"0.4.0-dev+1387"
|
||||
typealias AbstractString String
|
||||
end
|
||||
if VERSION < v"0.4.0-dev+1419"
|
||||
const UInt32 = Uint32
|
||||
end
|
||||
if VERSION < v"0.4.0-dev+3874"
|
||||
Base.parse{T<:Integer}(::Type{T}, s::AbstractString) = parseint(T, s)
|
||||
end
|
||||
|
||||
CharWidths = Dict{Int,Int}()
|
||||
|
||||
#############################################################################
|
||||
# Use ../libutf8proc for category codes, rather than the one in Julia,
|
||||
# to minimize bootstrapping complexity when a new version of Unicode comes out.
|
||||
catcode(c) = ccall((:utf8proc_category,"../libutf8proc"), Cint, (Int32,), c)
|
||||
|
||||
# use Base.UTF8proc module to get category codes constants, since
|
||||
# we won't change these in utf8proc.
|
||||
import Base.UTF8proc
|
||||
|
||||
#############################################################################
|
||||
# Use a default width of 1 for all character categories that are
|
||||
# letter/symbol/number-like. This can be overriden by Unifont or UAX 11
|
||||
# below, but provides a useful nonzero fallback for new codepoints when
|
||||
# a new Unicode version has been released but Unifont hasn't been updated yet.
|
||||
|
||||
zerowidth = Set{Int}() # categories that may contain zero-width chars
|
||||
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CN)
|
||||
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_MN)
|
||||
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_MC)
|
||||
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_ME)
|
||||
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_SK)
|
||||
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_ZS)
|
||||
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_ZL)
|
||||
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_ZP)
|
||||
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CC)
|
||||
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CF)
|
||||
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CS)
|
||||
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CO)
|
||||
for c in 0x0000:0x110000
|
||||
if catcode(c) ∉ zerowidth
|
||||
CharWidths[c] = 1
|
||||
end
|
||||
end
|
||||
|
||||
#############################################################################
|
||||
# Widths from GNU Unifont
|
||||
|
||||
#Read sfdfile for character widths
|
||||
function parsesfd(filename::AbstractString, CharWidths::Dict{Int,Int}=Dict{Int,Int}())
|
||||
state=:seekchar
|
||||
lineno = 0
|
||||
codepoint = width = nothing
|
||||
for line in readlines(open(filename))
|
||||
lineno += 1
|
||||
if state==:seekchar #StartChar: nonmarkingreturn
|
||||
if contains(line, "StartChar: ")
|
||||
codepoint = nothing
|
||||
width = nothing
|
||||
state = :readdata
|
||||
end
|
||||
elseif state==:readdata #Encoding: 65538 -1 2, Width: 1024
|
||||
contains(line, "Encoding:") && (codepoint = parse(Int, split(line)[3]))
|
||||
contains(line, "Width:") && (width = parse(Int, split(line)[2]))
|
||||
if codepoint!=nothing && width!=nothing && codepoint >= 0
|
||||
w=div(width, 512) # 512 units to the en
|
||||
if w > 0
|
||||
# only add nonzero widths, since (1) the default is zero
|
||||
# and (2) this circumvents some apparent bugs in Unifont
|
||||
# (https://savannah.gnu.org/bugs/index.php?45395)
|
||||
CharWidths[codepoint] = w
|
||||
end
|
||||
state = :seekchar
|
||||
end
|
||||
end
|
||||
end
|
||||
CharWidths
|
||||
end
|
||||
CharWidths=parsesfd("unifont.sfd", CharWidths)
|
||||
CharWidths=parsesfd("unifont_upper.sfd", CharWidths)
|
||||
|
||||
#############################################################################
|
||||
# Widths from UAX #11: East Asian Width
|
||||
# .. these take precedence over the Unifont width for all codepoints
|
||||
# listed explicitly as wide/full/narrow/half-width
|
||||
|
||||
for line in readlines(open("EastAsianWidth.txt"))
|
||||
#Strip comments
|
||||
line[1] == '#' && continue
|
||||
precomment = split(line, '#')[1]
|
||||
#Parse code point range and width code
|
||||
tokens = split(precomment, ';')
|
||||
length(tokens) >= 2 || continue
|
||||
charrange = tokens[1]
|
||||
width = strip(tokens[2])
|
||||
#Parse code point range into Julia UnitRange
|
||||
rangetokens = split(charrange, "..")
|
||||
charstart = parse(UInt32, "0x"*rangetokens[1])
|
||||
charend = parse(UInt32, "0x"*rangetokens[length(rangetokens)>1 ? 2 : 1])
|
||||
|
||||
#Assign widths
|
||||
for c in charstart:charend
|
||||
if width=="W" || width=="F" # wide or full
|
||||
CharWidths[c]=2
|
||||
elseif width=="Na"|| width=="H" # narrow or half
|
||||
CharWidths[c]=1
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
#############################################################################
|
||||
# A few exceptions to the above cases, found by manual comparison
|
||||
# to other wcwidth functions and similar checks.
|
||||
|
||||
for c in keys(CharWidths)
|
||||
cat = catcode(c)
|
||||
|
||||
# make sure format control character (category Cf) have width 0,
|
||||
# except for the Arabic characters 0x06xx (see unicode std 6.2, sec. 8.2)
|
||||
if cat==UTF8proc.UTF8PROC_CATEGORY_CF && c ∉ [0x0601,0x0602,0x0603,0x06dd]
|
||||
CharWidths[c]=0
|
||||
end
|
||||
|
||||
# Unifont has nonzero width for a number of non-spacing combining
|
||||
# characters, e.g. (in 7.0.06): f84,17b4,17b5,180b,180d,2d7f, and
|
||||
# the variation selectors
|
||||
if cat==UTF8proc.UTF8PROC_CATEGORY_MN
|
||||
CharWidths[c]=0
|
||||
end
|
||||
|
||||
# We also assign width of zero to unassigned and private-use
|
||||
# codepoints (Unifont includes ConScript Unicode Registry PUA fonts,
|
||||
# but since these are nonstandard it seems questionable to recognize them).
|
||||
if cat==UTF8proc.UTF8PROC_CATEGORY_CO || cat==UTF8proc.UTF8PROC_CATEGORY_CN
|
||||
CharWidths[c]=0
|
||||
end
|
||||
|
||||
# for some reason, Unifont has width-2 glyphs for ASCII control chars
|
||||
if cat==UTF8proc.UTF8PROC_CATEGORY_CC
|
||||
CharWidths[c]=0
|
||||
end
|
||||
end
|
||||
|
||||
#By definition, should have zero width (on the same line)
|
||||
#0x002028 '
' category: Zl name: LINE SEPARATOR/
|
||||
#0x002029 '
' category: Zp name: PARAGRAPH SEPARATOR/
|
||||
CharWidths[0x2028]=0
|
||||
CharWidths[0x2029]=0
|
||||
|
||||
#By definition, should be narrow = width of 1 en space
|
||||
#0x00202f ' ' category: Zs name: NARROW NO-BREAK SPACE/
|
||||
CharWidths[0x202f]=1
|
||||
|
||||
#By definition, should be wide = width of 1 em space
|
||||
#0x002001 ' ' category: Zs name: EM QUAD/
|
||||
#0x002003 ' ' category: Zs name: EM SPACE/
|
||||
CharWidths[0x2001]=2
|
||||
CharWidths[0x2003]=2
|
||||
|
||||
#############################################################################
|
||||
# Output (to a file or pipe) for processing by data_generator.rb
|
||||
# ... don't bother to output zero widths since that will be the default.
|
||||
|
||||
firstc = 0x000000
|
||||
lastv = 0
|
||||
uhex(c) = uppercase(hex(c,4))
|
||||
for c in 0x0000:0x110000
|
||||
v = get(CharWidths, c, 0)
|
||||
if v != lastv || c == 0x110000
|
||||
v < 4 || error("invalid charwidth $v for $c")
|
||||
if firstc+1 < c
|
||||
println(uhex(firstc), "..", uhex(c-1), "; ", lastv)
|
||||
else
|
||||
println(uhex(firstc), "; ", lastv)
|
||||
end
|
||||
firstc = c
|
||||
lastv = v
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,411 @@
|
|||
#!/usr/bin/env ruby
|
||||
|
||||
# This file was used to generate the 'unicode_data.c' file by parsing the
|
||||
# Unicode data file 'UnicodeData.txt' of the Unicode Character Database.
|
||||
# It is included for informational purposes only and not intended for
|
||||
# production use.
|
||||
|
||||
|
||||
# Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a
|
||||
# copy of this software and associated documentation files (the "Software"),
|
||||
# to deal in the Software without restriction, including without limitation
|
||||
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
# and/or sell copies of the Software, and to permit persons to whom the
|
||||
# Software is furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
# DEALINGS IN THE SOFTWARE.
|
||||
|
||||
|
||||
# This file contains derived data from a modified version of the
|
||||
# Unicode data files. The following license applies to that data:
|
||||
#
|
||||
# COPYRIGHT AND PERMISSION NOTICE
|
||||
#
|
||||
# Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed
|
||||
# under the Terms of Use in http://www.unicode.org/copyright.html.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a
|
||||
# copy of the Unicode data files and any associated documentation (the "Data
|
||||
# Files") or Unicode software and any associated documentation (the
|
||||
# "Software") to deal in the Data Files or Software without restriction,
|
||||
# including without limitation the rights to use, copy, modify, merge,
|
||||
# publish, distribute, and/or sell copies of the Data Files or Software, and
|
||||
# to permit persons to whom the Data Files or Software are furnished to do
|
||||
# so, provided that (a) the above copyright notice(s) and this permission
|
||||
# notice appear with all copies of the Data Files or Software, (b) both the
|
||||
# above copyright notice(s) and this permission notice appear in associated
|
||||
# documentation, and (c) there is clear notice in each modified Data File or
|
||||
# in the Software as well as in the documentation associated with the Data
|
||||
# File(s) or Software that the data or software has been modified.
|
||||
#
|
||||
# THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
|
||||
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
|
||||
# THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS
|
||||
# INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR
|
||||
# CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF
|
||||
# USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
|
||||
# TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
# PERFORMANCE OF THE DATA FILES OR SOFTWARE.
|
||||
#
|
||||
# Except as contained in this notice, the name of a copyright holder shall
|
||||
# not be used in advertising or otherwise to promote the sale, use or other
|
||||
# dealings in these Data Files or Software without prior written
|
||||
# authorization of the copyright holder.
|
||||
|
||||
|
||||
$ignorable_list = File.read("DerivedCoreProperties.txt")[/# Derived Property: Default_Ignorable_Code_Point.*?# Total code points:/m]
|
||||
$ignorable = []
|
||||
$ignorable_list.each_line do |entry|
|
||||
if entry =~ /^([0-9A-F]+)\.\.([0-9A-F]+)/
|
||||
$1.hex.upto($2.hex) { |e2| $ignorable << e2 }
|
||||
elsif entry =~ /^[0-9A-F]+/
|
||||
$ignorable << $&.hex
|
||||
end
|
||||
end
|
||||
|
||||
$grapheme_boundclass_list = File.read("GraphemeBreakProperty.txt")
|
||||
$grapheme_boundclass = Hash.new("UTF8PROC_BOUNDCLASS_OTHER")
|
||||
$grapheme_boundclass_list.each_line do |entry|
|
||||
if entry =~ /^([0-9A-F]+)\.\.([0-9A-F]+)\s*;\s*([A-Za-z_]+)/
|
||||
$1.hex.upto($2.hex) { |e2| $grapheme_boundclass[e2] = "UTF8PROC_BOUNDCLASS_" + $3.upcase }
|
||||
elsif entry =~ /^([0-9A-F]+)\s*;\s*([A-Za-z_]+)/
|
||||
$grapheme_boundclass[$1.hex] = "UTF8PROC_BOUNDCLASS_" + $2.upcase
|
||||
end
|
||||
end
|
||||
|
||||
$charwidth_list = File.read("CharWidths.txt")
|
||||
$charwidth = Hash.new(0)
|
||||
$charwidth_list.each_line do |entry|
|
||||
if entry =~ /^([0-9A-F]+)\.\.([0-9A-F]+)\s*;\s*([0-9]+)/
|
||||
$1.hex.upto($2.hex) { |e2| $charwidth[e2] = $3.to_i }
|
||||
elsif entry =~ /^([0-9A-F]+)\s*;\s*([0-9]+)/
|
||||
$charwidth[$1.hex] = $2.to_i
|
||||
end
|
||||
end
|
||||
|
||||
$exclusions = File.read("CompositionExclusions.txt")[/# \(1\) Script Specifics.*?# Total code points:/m]
|
||||
$exclusions = $exclusions.chomp.split("\n").collect { |e| e.hex }
|
||||
|
||||
$excl_version = File.read("CompositionExclusions.txt")[/# \(2\) Post Composition Version precomposed characters.*?# Total code points:/m]
|
||||
$excl_version = $excl_version.chomp.split("\n").collect { |e| e.hex }
|
||||
|
||||
$case_folding_string = File.open("CaseFolding.txt", :encoding => 'utf-8').read
|
||||
$case_folding = {}
|
||||
$case_folding_string.chomp.split("\n").each do |line|
|
||||
next unless line =~ /([0-9A-F]+); [CFS]; ([0-9A-F ]+);/i
|
||||
$case_folding[$1.hex] = $2.split(" ").collect { |e| e.hex }
|
||||
end
|
||||
|
||||
$int_array = []
|
||||
$int_array_indicies = {}
|
||||
|
||||
def str2c(string, prefix)
|
||||
return "0" if string.nil?
|
||||
return "UTF8PROC_#{prefix}_#{string.upcase}"
|
||||
end
|
||||
def pushary(array)
|
||||
idx = $int_array_indicies[array]
|
||||
unless idx
|
||||
$int_array_indicies[array] = $int_array.length
|
||||
idx = $int_array.length
|
||||
array.each { |entry| $int_array << entry }
|
||||
end
|
||||
return idx
|
||||
end
|
||||
def cpary2utf16encoded(array)
|
||||
return array.flat_map { |cp|
|
||||
if (cp <= 0xFFFF)
|
||||
raise "utf-16 code: #{cp}" if cp & 0b1111100000000000 == 0b1101100000000000
|
||||
cp
|
||||
else
|
||||
temp = cp - 0x10000
|
||||
[(temp >> 10) | 0b1101100000000000, (temp & 0b0000001111111111) | 0b1101110000000000]
|
||||
end
|
||||
}
|
||||
end
|
||||
def cpary2c(array)
|
||||
return "UINT16_MAX" if array.nil? || array.length == 0
|
||||
lencode = array.length - 1 #no sequence has len 0, so we encode len 1 as 0, len 2 as 1, ...
|
||||
array = cpary2utf16encoded(array)
|
||||
if lencode >= 7 #we have only 3 bits for the length (which is already cutting it close. might need to change it to 2 bits in future Unicode versions)
|
||||
array = [lencode] + array
|
||||
lencode = 7
|
||||
end
|
||||
idx = pushary(array)
|
||||
raise "Array index out of bound" if idx > 0x1FFF
|
||||
return "#{idx | (lencode << 13)}"
|
||||
end
|
||||
def singlecpmap(cp)
|
||||
return "UINT16_MAX" if cp == nil
|
||||
idx = pushary(cpary2utf16encoded([cp]))
|
||||
raise "Array index out of bound" if idx > 0xFFFF
|
||||
return "#{idx}"
|
||||
end
|
||||
|
||||
class UnicodeChar
|
||||
attr_accessor :code, :name, :category, :combining_class, :bidi_class,
|
||||
:decomp_type, :decomp_mapping,
|
||||
:bidi_mirrored,
|
||||
:uppercase_mapping, :lowercase_mapping, :titlecase_mapping,
|
||||
#caches:
|
||||
:c_entry_index, :c_decomp_mapping, :c_case_folding
|
||||
def initialize(line)
|
||||
raise "Could not parse input." unless line =~ /^
|
||||
([0-9A-F]+); # code
|
||||
([^;]+); # name
|
||||
([A-Z]+); # general category
|
||||
([0-9]+); # canonical combining class
|
||||
([A-Z]+); # bidi class
|
||||
(<([A-Z]*)>)? # decomposition type
|
||||
((\ ?[0-9A-F]+)*); # decompomposition mapping
|
||||
([0-9]*); # decimal digit
|
||||
([0-9]*); # digit
|
||||
([^;]*); # numeric
|
||||
([YN]*); # bidi mirrored
|
||||
([^;]*); # unicode 1.0 name
|
||||
([^;]*); # iso comment
|
||||
([0-9A-F]*); # simple uppercase mapping
|
||||
([0-9A-F]*); # simple lowercase mapping
|
||||
([0-9A-F]*)$/ix # simple titlecase mapping
|
||||
@code = $1.hex
|
||||
@name = $2
|
||||
@category = $3
|
||||
@combining_class = Integer($4)
|
||||
@bidi_class = $5
|
||||
@decomp_type = $7
|
||||
@decomp_mapping = ($8=='') ? nil :
|
||||
$8.split.collect { |element| element.hex }
|
||||
@bidi_mirrored = ($13=='Y') ? true : false
|
||||
@uppercase_mapping = ($16=='') ? nil : $16.hex
|
||||
@lowercase_mapping = ($17=='') ? nil : $17.hex
|
||||
@titlecase_mapping = ($18=='') ? nil : $18.hex
|
||||
end
|
||||
def case_folding
|
||||
$case_folding[code]
|
||||
end
|
||||
def c_entry(comb_indicies)
|
||||
" " <<
|
||||
"{#{str2c category, 'CATEGORY'}, #{combining_class}, " <<
|
||||
"#{str2c bidi_class, 'BIDI_CLASS'}, " <<
|
||||
"#{str2c decomp_type, 'DECOMP_TYPE'}, " <<
|
||||
"#{c_decomp_mapping}, " <<
|
||||
"#{c_case_folding}, " <<
|
||||
"#{singlecpmap uppercase_mapping }, " <<
|
||||
"#{singlecpmap lowercase_mapping }, " <<
|
||||
"#{singlecpmap titlecase_mapping }, " <<
|
||||
"#{comb_indicies[code] ? comb_indicies[code]: 'UINT16_MAX'}, " <<
|
||||
"#{bidi_mirrored}, " <<
|
||||
"#{$exclusions.include?(code) or $excl_version.include?(code)}, " <<
|
||||
"#{$ignorable.include?(code)}, " <<
|
||||
"#{%W[Zl Zp Cc Cf].include?(category) and not [0x200C, 0x200D].include?(category)}, " <<
|
||||
"#{$charwidth[code]}, 0, " <<
|
||||
"#{$grapheme_boundclass[code]}},\n"
|
||||
end
|
||||
end
|
||||
|
||||
chars = []
|
||||
char_hash = {}
|
||||
|
||||
while gets
|
||||
if $_ =~ /^([0-9A-F]+);<[^;>,]+, First>;/i
|
||||
first = $1.hex
|
||||
gets
|
||||
char = UnicodeChar.new($_)
|
||||
raise "No last character of sequence found." unless
|
||||
$_ =~ /^([0-9A-F]+);<([^;>,]+), Last>;/i
|
||||
last = $1.hex
|
||||
name = "<#{$2}>"
|
||||
for i in first..last
|
||||
char_clone = char.clone
|
||||
char_clone.code = i
|
||||
char_clone.name = name
|
||||
char_hash[char_clone.code] = char_clone
|
||||
chars << char_clone
|
||||
end
|
||||
else
|
||||
char = UnicodeChar.new($_)
|
||||
char_hash[char.code] = char
|
||||
chars << char
|
||||
end
|
||||
end
|
||||
|
||||
comb1st_indicies = {}
|
||||
comb2nd_indicies = {}
|
||||
comb2nd_indicies_sorted_keys = []
|
||||
comb2nd_indicies_nonbasic = {}
|
||||
comb_array = []
|
||||
|
||||
chars.each do |char|
|
||||
if !char.nil? and char.decomp_type.nil? and char.decomp_mapping and
|
||||
char.decomp_mapping.length == 2 and !char_hash[char.decomp_mapping[0]].nil? and
|
||||
char_hash[char.decomp_mapping[0]].combining_class == 0 and
|
||||
not $exclusions.include?(char.code)
|
||||
|
||||
dm0 = char.decomp_mapping[0]
|
||||
dm1 = char.decomp_mapping[1]
|
||||
unless comb1st_indicies[dm0]
|
||||
comb1st_indicies[dm0] = comb1st_indicies.keys.length
|
||||
end
|
||||
unless comb2nd_indicies[dm1]
|
||||
comb2nd_indicies_sorted_keys << dm1
|
||||
comb2nd_indicies[dm1] = comb2nd_indicies.keys.length
|
||||
end
|
||||
comb_array[comb1st_indicies[dm0]] ||= []
|
||||
raise "Duplicate canonical mapping: #{char.code} #{dm0} #{dm1}" if comb_array[comb1st_indicies[dm0]][comb2nd_indicies[dm1]]
|
||||
comb_array[comb1st_indicies[dm0]][comb2nd_indicies[dm1]] = char.code
|
||||
|
||||
comb2nd_indicies_nonbasic[dm1] = true if char.code > 0xFFFF
|
||||
end
|
||||
char.c_decomp_mapping = cpary2c(char.decomp_mapping)
|
||||
char.c_case_folding = cpary2c(char.case_folding)
|
||||
end
|
||||
|
||||
comb_indicies = {}
|
||||
cumoffset = 0
|
||||
comb1st_indicies_lastoffsets = []
|
||||
comb1st_indicies_firstoffsets = []
|
||||
comb1st_indicies.each do |dm0, index|
|
||||
first = nil
|
||||
last = nil
|
||||
offset = 0
|
||||
comb2nd_indicies_sorted_keys.each_with_index do |dm1, b|
|
||||
if comb_array[index][b]
|
||||
first = offset unless first
|
||||
last = offset
|
||||
last += 1 if comb2nd_indicies_nonbasic[dm1]
|
||||
end
|
||||
offset += 1
|
||||
offset += 1 if comb2nd_indicies_nonbasic[dm1]
|
||||
end
|
||||
comb1st_indicies_firstoffsets[index] = first
|
||||
comb1st_indicies_lastoffsets[index] = last
|
||||
raise "double index" if comb_indicies[dm0]
|
||||
comb_indicies[dm0] = cumoffset
|
||||
cumoffset += last - first + 1 + 2
|
||||
end
|
||||
|
||||
offset = 0
|
||||
comb2nd_indicies_sorted_keys.each do |dm1|
|
||||
raise "double index" if comb_indicies[dm1]
|
||||
comb_indicies[dm1] = 0x8000 | (comb2nd_indicies[dm1] + offset)
|
||||
raise "too large comb index" if comb2nd_indicies[dm1] + offset > 0x4000
|
||||
if comb2nd_indicies_nonbasic[dm1]
|
||||
comb_indicies[dm1] = comb_indicies[dm1] | 0x4000
|
||||
offset += 1
|
||||
end
|
||||
end
|
||||
|
||||
properties_indicies = {}
|
||||
properties = []
|
||||
chars.each do |char|
|
||||
c_entry = char.c_entry(comb_indicies)
|
||||
char.c_entry_index = properties_indicies[c_entry]
|
||||
unless char.c_entry_index
|
||||
properties_indicies[c_entry] = properties.length
|
||||
char.c_entry_index = properties.length
|
||||
properties << c_entry
|
||||
end
|
||||
end
|
||||
|
||||
stage1 = []
|
||||
stage2 = []
|
||||
for code in 0...0x110000
|
||||
next unless code % 0x100 == 0
|
||||
stage2_entry = []
|
||||
for code2 in code...(code+0x100)
|
||||
if char_hash[code2]
|
||||
stage2_entry << (char_hash[code2].c_entry_index + 1)
|
||||
else
|
||||
stage2_entry << 0
|
||||
end
|
||||
end
|
||||
old_index = stage2.index(stage2_entry)
|
||||
if old_index
|
||||
stage1 << (old_index * 0x100)
|
||||
else
|
||||
stage1 << (stage2.length * 0x100)
|
||||
stage2 << stage2_entry
|
||||
end
|
||||
end
|
||||
|
||||
$stdout << "static const utf8proc_uint16_t utf8proc_sequences[] = {\n "
|
||||
i = 0
|
||||
$int_array.each do |entry|
|
||||
i += 1
|
||||
if i == 8
|
||||
i = 0
|
||||
$stdout << "\n "
|
||||
end
|
||||
$stdout << entry << ", "
|
||||
end
|
||||
$stdout << "};\n\n"
|
||||
|
||||
$stdout << "static const utf8proc_uint16_t utf8proc_stage1table[] = {\n "
|
||||
i = 0
|
||||
stage1.each do |entry|
|
||||
i += 1
|
||||
if i == 8
|
||||
i = 0
|
||||
$stdout << "\n "
|
||||
end
|
||||
$stdout << entry << ", "
|
||||
end
|
||||
$stdout << "};\n\n"
|
||||
|
||||
$stdout << "static const utf8proc_uint16_t utf8proc_stage2table[] = {\n "
|
||||
i = 0
|
||||
stage2.flatten.each do |entry|
|
||||
i += 1
|
||||
if i == 8
|
||||
i = 0
|
||||
$stdout << "\n "
|
||||
end
|
||||
$stdout << entry << ", "
|
||||
end
|
||||
$stdout << "};\n\n"
|
||||
|
||||
$stdout << "static const utf8proc_property_t utf8proc_properties[] = {\n"
|
||||
$stdout << " {0, 0, 0, 0, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, false,false,false,false, 0, 0, UTF8PROC_BOUNDCLASS_OTHER},\n"
|
||||
properties.each { |line|
|
||||
$stdout << line
|
||||
}
|
||||
$stdout << "};\n\n"
|
||||
|
||||
|
||||
|
||||
$stdout << "static const utf8proc_uint16_t utf8proc_combinations[] = {\n "
|
||||
i = 0
|
||||
comb1st_indicies.keys.each_index do |a|
|
||||
offset = 0
|
||||
$stdout << comb1st_indicies_firstoffsets[a] << ", " << comb1st_indicies_lastoffsets[a] << ", "
|
||||
comb2nd_indicies_sorted_keys.each_with_index do |dm1, b|
|
||||
break if offset > comb1st_indicies_lastoffsets[a]
|
||||
if offset >= comb1st_indicies_firstoffsets[a]
|
||||
i += 1
|
||||
if i == 8
|
||||
i = 0
|
||||
$stdout << "\n "
|
||||
end
|
||||
v = comb_array[a][b] ? comb_array[a][b] : 0
|
||||
$stdout << (( v & 0xFFFF0000 ) >> 16) << ", " if comb2nd_indicies_nonbasic[dm1]
|
||||
$stdout << (v & 0xFFFF) << ", "
|
||||
end
|
||||
offset += 1
|
||||
offset += 1 if comb2nd_indicies_nonbasic[dm1]
|
||||
end
|
||||
$stdout << "\n"
|
||||
end
|
||||
$stdout << "};\n\n"
|
||||
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
```
|
||||
U+0020 <-- all space characters (general category Zs)
|
||||
U+0027 ' <-- left/right single quotation mark U+2018..2019,
|
||||
modifier letter apostrophe U+02BC,
|
||||
modifier letter vertical line U+02C8
|
||||
U+002D - <-- all dash characters (general category Pd),
|
||||
minus U+2212
|
||||
U+002F / <-- fraction slash U+2044,
|
||||
division slash U+2215
|
||||
U+003A : <-- ratio U+2236
|
||||
U+003C < <-- single left-pointing angle quotation mark U+2039,
|
||||
left-pointing angle bracket U+2329,
|
||||
left angle bracket U+3008
|
||||
U+003E > <-- single right-pointing angle quotation mark U+203A,
|
||||
right-pointing angle bracket U+232A,
|
||||
right angle bracket U+3009
|
||||
U+005C \ <-- set minus U+2216
|
||||
U+005E ^ <-- modifier letter up arrowhead U+02C4,
|
||||
modifier letter circumflex accent U+02C6,
|
||||
caret U+2038,
|
||||
up arrowhead U+2303
|
||||
U+005F _ <-- all connector characters (general category Pc),
|
||||
modifier letter low macron U+02CD
|
||||
U+0060 ` <-- modifier letter grave accent U+02CB
|
||||
U+007C | <-- divides U+2223
|
||||
U+007E ~ <-- tilde operator U+223C
|
||||
```
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
#include "tests.h"
|
||||
#include <wctype.h>
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
int error = 0, better = 0;
|
||||
utf8proc_int32_t c;
|
||||
|
||||
(void) argc; /* unused */
|
||||
(void) argv; /* unused */
|
||||
|
||||
/* some simple sanity tests of the character widths */
|
||||
for (c = 0; c <= 0x110000; ++c) {
|
||||
utf8proc_int32_t l = utf8proc_tolower(c);
|
||||
utf8proc_int32_t u = utf8proc_toupper(c);
|
||||
|
||||
check(l == c || utf8proc_codepoint_valid(l), "invalid tolower");
|
||||
check(u == c || utf8proc_codepoint_valid(u), "invalid toupper");
|
||||
|
||||
if (sizeof(wint_t) > 2 || c < (1<<16)) {
|
||||
wint_t l0 = towlower(c), u0 = towupper(c);
|
||||
|
||||
/* OS unicode tables may be out of date. But if they
|
||||
do have a lower/uppercase mapping, hopefully it
|
||||
is correct? */
|
||||
if (l0 != c && l0 != l) {
|
||||
fprintf(stderr, "MISMATCH %x != towlower(%x) == %x\n",
|
||||
l, c, l0);
|
||||
++error;
|
||||
}
|
||||
else if (l0 != l) { /* often true for out-of-date OS unicode */
|
||||
++better;
|
||||
/* printf("%x != towlower(%x) == %x\n", l, c, l0); */
|
||||
}
|
||||
if (u0 != c && u0 != u) {
|
||||
fprintf(stderr, "MISMATCH %x != towupper(%x) == %x\n",
|
||||
u, c, u0);
|
||||
++error;
|
||||
}
|
||||
else if (u0 != u) { /* often true for out-of-date OS unicode */
|
||||
++better;
|
||||
/* printf("%x != towupper(%x) == %x\n", u, c, u0); */
|
||||
}
|
||||
}
|
||||
}
|
||||
check(!error, "utf8proc case conversion FAILED %d tests.", error);
|
||||
printf("More up-to-date than OS unicode tables for %d tests.\n", better);
|
||||
printf("utf8proc case conversion tests SUCCEEDED.\n");
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
#include "tests.h"
|
||||
#include <ctype.h>
|
||||
#include <wchar.h>
|
||||
|
||||
static int my_isprint(int c) {
|
||||
int cat = utf8proc_get_property(c)->category;
|
||||
return (UTF8PROC_CATEGORY_LU <= cat && cat <= UTF8PROC_CATEGORY_ZS) ||
|
||||
(c == 0x0601 || c == 0x0602 || c == 0x0603 || c == 0x06dd);
|
||||
}
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
int c, error = 0, updates = 0;
|
||||
|
||||
(void) argc; /* unused */
|
||||
(void) argv; /* unused */
|
||||
|
||||
/* some simple sanity tests of the character widths */
|
||||
for (c = 0; c <= 0x110000; ++c) {
|
||||
int cat = utf8proc_get_property(c)->category;
|
||||
int w = utf8proc_charwidth(c);
|
||||
if ((cat == UTF8PROC_CATEGORY_MN || cat == UTF8PROC_CATEGORY_ME) &&
|
||||
w > 0) {
|
||||
fprintf(stderr, "nonzero width %d for combining char %x\n", w, c);
|
||||
error = 1;
|
||||
}
|
||||
if (w == 0 &&
|
||||
((cat >= UTF8PROC_CATEGORY_LU && cat <= UTF8PROC_CATEGORY_LO) ||
|
||||
(cat >= UTF8PROC_CATEGORY_ND && cat <= UTF8PROC_CATEGORY_SC) ||
|
||||
(cat >= UTF8PROC_CATEGORY_SO && cat <= UTF8PROC_CATEGORY_ZS))) {
|
||||
fprintf(stderr, "zero width for symbol-like char %x\n", c);
|
||||
error = 1;
|
||||
}
|
||||
if (c <= 127 && ((!isprint(c) && w > 0) ||
|
||||
(isprint(c) && wcwidth(c) != w))) {
|
||||
fprintf(stderr, "wcwidth %d mismatch %d for %s ASCII %x\n",
|
||||
wcwidth(c), w,
|
||||
isprint(c) ? "printable" : "non-printable", c);
|
||||
error = 1;
|
||||
}
|
||||
if (!my_isprint(c) && w > 0) {
|
||||
fprintf(stderr, "non-printing %x had width %d\n", c, w);
|
||||
error = 1;
|
||||
}
|
||||
}
|
||||
check(!error, "utf8proc_charwidth FAILED tests.");
|
||||
|
||||
/* print some other information by compariing with system wcwidth */
|
||||
printf("Mismatches with system wcwidth (not necessarily errors):\n");
|
||||
for (c = 0; c <= 0x110000; ++c) {
|
||||
int w = utf8proc_charwidth(c);
|
||||
int wc = wcwidth(c);
|
||||
if (sizeof(wchar_t) == 2 && c >= (1<<16)) continue;
|
||||
/* lots of these errors for out-of-date system unicode tables */
|
||||
if (wc == -1 && my_isprint(c) && w > 0) {
|
||||
updates += 1;
|
||||
#if 0
|
||||
printf(" wcwidth(%x) = -1 for printable char\n", c);
|
||||
#endif
|
||||
}
|
||||
if (wc == -1 && !my_isprint(c) && w > 0)
|
||||
printf(" wcwidth(%x) = -1 for non-printable width-%d char\n", c, w);
|
||||
if (wc >= 0 && wc != w)
|
||||
printf(" wcwidth(%x) = %d != charwidth %d\n", c, wc, w);
|
||||
}
|
||||
printf(" ... (positive widths for %d chars unknown to wcwidth) ...\n",
|
||||
updates);
|
||||
printf("Character-width tests SUCCEEDED.\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
#include "tests.h"
|
||||
|
||||
static int thunk_test = 1;
|
||||
|
||||
static utf8proc_int32_t custom(utf8proc_int32_t codepoint, void *thunk)
|
||||
{
|
||||
check(((int *) thunk) == &thunk_test, "unexpected thunk passed");
|
||||
if (codepoint == 'a')
|
||||
return 'b';
|
||||
if (codepoint == 'S')
|
||||
return 0x00df; /* ß */
|
||||
return codepoint;
|
||||
}
|
||||
|
||||
int main(void)
|
||||
{
|
||||
utf8proc_uint8_t input[] = {0x41,0x61,0x53,0x62,0xef,0xbd,0x81,0x00}; /* "AaSb\uff41" */
|
||||
utf8proc_uint8_t correct[] = {0x61,0x62,0x73,0x73,0x62,0x61,0x00}; /* "abssba" */
|
||||
utf8proc_uint8_t *output;
|
||||
utf8proc_map_custom(input, 0, &output, UTF8PROC_CASEFOLD | UTF8PROC_COMPOSE | UTF8PROC_COMPAT | UTF8PROC_NULLTERM,
|
||||
custom, &thunk_test);
|
||||
printf("mapped \"%s\" -> \"%s\"\n", (char*)input, (char*)output);
|
||||
check(strlen((char*) output) == 6, "incorrect output length");
|
||||
check(!memcmp(correct, output, 7), "incorrect output data");
|
||||
free(output);
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
#include "tests.h"
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
char *buf = NULL;
|
||||
size_t bufsize = 0;
|
||||
FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL;
|
||||
utf8proc_uint8_t src[1024];
|
||||
int len;
|
||||
|
||||
check(f != NULL, "error opening GraphemeBreakTest.txt");
|
||||
while (getline(&buf, &bufsize, f) > 0) {
|
||||
size_t bi = 0, si = 0;
|
||||
lineno += 1;
|
||||
|
||||
if (lineno % 100 == 0)
|
||||
printf("checking line %zd...\n", lineno);
|
||||
|
||||
if (buf[0] == '#') continue;
|
||||
|
||||
while (buf[bi]) {
|
||||
bi = skipspaces(buf, bi);
|
||||
if (buf[bi] == '/') { /* grapheme break */
|
||||
src[si++] = '/';
|
||||
bi++;
|
||||
}
|
||||
else if (buf[bi] == '+') { /* no break */
|
||||
bi++;
|
||||
}
|
||||
else if (buf[bi] == '#') { /* start of comments */
|
||||
break;
|
||||
}
|
||||
else { /* hex-encoded codepoint */
|
||||
len = encode((char*) (src + si), buf + bi) - 1;
|
||||
while (src[si]) ++si; /* advance to NUL termination */
|
||||
bi += len;
|
||||
}
|
||||
}
|
||||
if (si && src[si-1] == '/')
|
||||
--si; /* no break after final grapheme */
|
||||
src[si] = 0; /* NUL-terminate */
|
||||
|
||||
if (si) {
|
||||
utf8proc_uint8_t utf8[1024]; /* copy src without 0xff grapheme separators */
|
||||
size_t i = 0, j = 0;
|
||||
utf8proc_ssize_t glen;
|
||||
utf8proc_uint8_t *g; /* utf8proc_map grapheme results */
|
||||
while (i < si) {
|
||||
if (src[i] != '/')
|
||||
utf8[j++] = src[i++];
|
||||
else
|
||||
i++;
|
||||
}
|
||||
glen = utf8proc_map(utf8, j, &g, UTF8PROC_CHARBOUND);
|
||||
if (glen == UTF8PROC_ERROR_INVALIDUTF8) {
|
||||
/* the test file contains surrogate codepoints, which are only for UTF-16 */
|
||||
printf("line %zd: ignoring invalid UTF-8 codepoints\n", lineno);
|
||||
}
|
||||
else {
|
||||
check(glen >= 0, "utf8proc_map error = %s",
|
||||
utf8proc_errmsg(glen));
|
||||
for (i = 0; i <= glen; ++i)
|
||||
if (g[i] == 0xff)
|
||||
g[i] = '/'; /* easier-to-read output (/ is not in test strings) */
|
||||
check(!strcmp((char*)g, (char*)src),
|
||||
"grapheme mismatch: \"%s\" instead of \"%s\"", (char*)g, (char*)src);
|
||||
}
|
||||
free(g);
|
||||
}
|
||||
}
|
||||
fclose(f);
|
||||
printf("Passed tests after %zd lines!\n", lineno);
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -0,0 +1,166 @@
|
|||
#include "tests.h"
|
||||
#include <ctype.h>
|
||||
#include <wchar.h>
|
||||
|
||||
static int tests;
|
||||
static int error;
|
||||
|
||||
#define CHECKVALID(pos, val, len) buf[pos] = val; testbytes(buf,len,len,__LINE__)
|
||||
#define CHECKINVALID(pos, val, len) buf[pos] = val; testbytes(buf,len,UTF8PROC_ERROR_INVALIDUTF8,__LINE__)
|
||||
|
||||
static void testbytes(unsigned char *buf, int len, utf8proc_ssize_t retval, int line)
|
||||
{
|
||||
utf8proc_int32_t out[16];
|
||||
utf8proc_ssize_t ret;
|
||||
|
||||
/* Make a copy to ensure that memory is left uninitialized after "len"
|
||||
* bytes. This way, Valgrind can detect overreads.
|
||||
*/
|
||||
unsigned char tmp[16];
|
||||
memcpy(tmp, buf, len);
|
||||
|
||||
tests++;
|
||||
if ((ret = utf8proc_iterate(tmp, len, out)) != retval) {
|
||||
fprintf(stderr, "Failed (%d):", line);
|
||||
for (int i = 0; i < len ; i++) {
|
||||
fprintf(stderr, " 0x%02x", tmp[i]);
|
||||
}
|
||||
fprintf(stderr, " -> %zd\n", ret);
|
||||
error++;
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
uint32_t byt;
|
||||
unsigned char buf[16];
|
||||
|
||||
tests = error = 0;
|
||||
|
||||
// Check valid sequences that were considered valid erroneously before
|
||||
buf[0] = 0xef;
|
||||
buf[1] = 0xb7;
|
||||
for (byt = 0x90; byt < 0xa0; byt++) {
|
||||
CHECKVALID(2, byt, 3);
|
||||
}
|
||||
// Check 0xfffe and 0xffff
|
||||
buf[1] = 0xbf;
|
||||
CHECKVALID(2, 0xbe, 3);
|
||||
CHECKVALID(2, 0xbf, 3);
|
||||
// Check 0x??fffe & 0x??ffff
|
||||
for (byt = 0x1fffe; byt < 0x110000; byt += 0x10000) {
|
||||
buf[0] = 0xf0 | (byt >> 18);
|
||||
buf[1] = 0x80 | ((byt >> 12) & 0x3f);
|
||||
CHECKVALID(3, 0xbe, 4);
|
||||
CHECKVALID(3, 0xbf, 4);
|
||||
}
|
||||
|
||||
// Continuation byte not after lead
|
||||
for (byt = 0x80; byt < 0xc0; byt++) {
|
||||
CHECKINVALID(0, byt, 1);
|
||||
}
|
||||
|
||||
// Continuation byte not after lead
|
||||
for (byt = 0x80; byt < 0xc0; byt++) {
|
||||
CHECKINVALID(0, byt, 1);
|
||||
}
|
||||
|
||||
// Test lead bytes
|
||||
for (byt = 0xc0; byt <= 0xff; byt++) {
|
||||
// Single lead byte at end of string
|
||||
CHECKINVALID(0, byt, 1);
|
||||
// Lead followed by non-continuation character < 0x80
|
||||
CHECKINVALID(1, 65, 2);
|
||||
// Lead followed by non-continuation character > 0xbf
|
||||
CHECKINVALID(1, 0xc0, 2);
|
||||
}
|
||||
|
||||
// Test overlong 2-byte
|
||||
buf[0] = 0xc0;
|
||||
for (byt = 0x81; byt <= 0xbf; byt++) {
|
||||
CHECKINVALID(1, byt, 2);
|
||||
}
|
||||
buf[0] = 0xc1;
|
||||
for (byt = 0x80; byt <= 0xbf; byt++) {
|
||||
CHECKINVALID(1, byt, 2);
|
||||
}
|
||||
|
||||
// Test overlong 3-byte
|
||||
buf[0] = 0xe0;
|
||||
buf[2] = 0x80;
|
||||
for (byt = 0x80; byt <= 0x9f; byt++) {
|
||||
CHECKINVALID(1, byt, 3);
|
||||
}
|
||||
|
||||
// Test overlong 4-byte
|
||||
buf[0] = 0xf0;
|
||||
buf[2] = 0x80;
|
||||
buf[3] = 0x80;
|
||||
for (byt = 0x80; byt <= 0x8f; byt++) {
|
||||
CHECKINVALID(1, byt, 4);
|
||||
}
|
||||
|
||||
// Test 4-byte > 0x10ffff
|
||||
buf[0] = 0xf4;
|
||||
buf[2] = 0x80;
|
||||
buf[3] = 0x80;
|
||||
for (byt = 0x90; byt <= 0xbf; byt++) {
|
||||
CHECKINVALID(1, byt, 4);
|
||||
}
|
||||
buf[1] = 0x80;
|
||||
for (byt = 0xf5; byt <= 0xf7; byt++) {
|
||||
CHECKINVALID(0, byt, 4);
|
||||
}
|
||||
|
||||
// Test 5-byte
|
||||
buf[4] = 0x80;
|
||||
for (byt = 0xf8; byt <= 0xfb; byt++) {
|
||||
CHECKINVALID(0, byt, 5);
|
||||
}
|
||||
|
||||
// Test 6-byte
|
||||
buf[5] = 0x80;
|
||||
for (byt = 0xfc; byt <= 0xfd; byt++) {
|
||||
CHECKINVALID(0, byt, 6);
|
||||
}
|
||||
|
||||
// Test 7-byte
|
||||
buf[6] = 0x80;
|
||||
CHECKINVALID(0, 0xfe, 7);
|
||||
|
||||
// Three and above byte sequences
|
||||
for (byt = 0xe0; byt < 0xf0; byt++) {
|
||||
// Lead followed by only 1 continuation byte
|
||||
CHECKINVALID(0, byt, 2);
|
||||
// Lead ended by non-continuation character < 0x80
|
||||
CHECKINVALID(2, 65, 3);
|
||||
// Lead ended by non-continuation character > 0xbf
|
||||
CHECKINVALID(2, 0xc0, 3);
|
||||
}
|
||||
|
||||
// 3-byte encoded surrogate character(s)
|
||||
buf[0] = 0xed; buf[2] = 0x80;
|
||||
// Single surrogate
|
||||
CHECKINVALID(1, 0xa0, 3);
|
||||
// Trailing surrogate first
|
||||
CHECKINVALID(1, 0xb0, 3);
|
||||
|
||||
// Four byte sequences
|
||||
buf[1] = 0x80;
|
||||
for (byt = 0xf0; byt < 0xf5; byt++) {
|
||||
// Lead followed by only 1 continuation bytes
|
||||
CHECKINVALID(0, byt, 2);
|
||||
// Lead followed by only 2 continuation bytes
|
||||
CHECKINVALID(0, byt, 3);
|
||||
// Lead followed by non-continuation character < 0x80
|
||||
CHECKINVALID(3, 65, 4);
|
||||
// Lead followed by non-continuation character > 0xbf
|
||||
CHECKINVALID(3, 0xc0, 4);
|
||||
|
||||
}
|
||||
|
||||
check(!error, "utf8proc_iterate FAILED %d tests out of %d", error, tests);
|
||||
printf("utf8proc_iterate tests SUCCEEDED, (%d) tests passed.\n", tests);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
#include "tests.h"
|
||||
|
||||
#define CHECK_NORM(NRM, norm, src) { \
|
||||
char *src_norm = (char*) utf8proc_ ## NRM((utf8proc_uint8_t*) src); \
|
||||
check(!strcmp(norm, src_norm), \
|
||||
"normalization failed for %s -> %s", src, norm); \
|
||||
free(src_norm); \
|
||||
}
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
char *buf = NULL;
|
||||
size_t bufsize = 0;
|
||||
FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL;
|
||||
char source[1024], NFC[1024], NFD[1024], NFKC[1024], NFKD[1024];
|
||||
|
||||
check(f != NULL, "error opening NormalizationTest.txt");
|
||||
while (getline(&buf, &bufsize, f) > 0) {
|
||||
size_t offset;
|
||||
lineno += 1;
|
||||
|
||||
if (buf[0] == '@') {
|
||||
printf("line %zd: %s", lineno, buf + 1);
|
||||
continue;
|
||||
}
|
||||
else if (lineno % 1000 == 0)
|
||||
printf("checking line %zd...\n", lineno);
|
||||
|
||||
if (buf[0] == '#') continue;
|
||||
|
||||
offset = encode(source, buf);
|
||||
offset += encode(NFC, buf + offset);
|
||||
offset += encode(NFD, buf + offset);
|
||||
offset += encode(NFKC, buf + offset);
|
||||
offset += encode(NFKD, buf + offset);
|
||||
|
||||
CHECK_NORM(NFC, NFC, source);
|
||||
CHECK_NORM(NFC, NFC, NFC);
|
||||
CHECK_NORM(NFC, NFC, NFD);
|
||||
CHECK_NORM(NFC, NFKC, NFKC);
|
||||
CHECK_NORM(NFC, NFKC, NFKD);
|
||||
|
||||
CHECK_NORM(NFD, NFD, source);
|
||||
CHECK_NORM(NFD, NFD, NFC);
|
||||
CHECK_NORM(NFD, NFD, NFD);
|
||||
CHECK_NORM(NFD, NFKD, NFKC);
|
||||
CHECK_NORM(NFD, NFKD, NFKD);
|
||||
|
||||
CHECK_NORM(NFKC, NFKC, source);
|
||||
CHECK_NORM(NFKC, NFKC, NFC);
|
||||
CHECK_NORM(NFKC, NFKC, NFD);
|
||||
CHECK_NORM(NFKC, NFKC, NFKC);
|
||||
CHECK_NORM(NFKC, NFKC, NFKD);
|
||||
|
||||
CHECK_NORM(NFKD, NFKD, source);
|
||||
CHECK_NORM(NFKD, NFKD, NFC);
|
||||
CHECK_NORM(NFKD, NFKD, NFD);
|
||||
CHECK_NORM(NFKD, NFKD, NFKC);
|
||||
CHECK_NORM(NFKD, NFKD, NFKD);
|
||||
}
|
||||
fclose(f);
|
||||
printf("Passed tests after %zd lines!\n", lineno);
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
/* simple test program to print out the utf8proc properties for a codepoint */
|
||||
|
||||
#include "tests.h"
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 1; i < argc; ++i) {
|
||||
unsigned int c;
|
||||
if (!strcmp(argv[i], "-V")) {
|
||||
printf("utf8proc version %s\n", utf8proc_version());
|
||||
continue;
|
||||
}
|
||||
check(sscanf(argv[i],"%x",&c) == 1, "invalid hex input %s", argv[i]);
|
||||
const utf8proc_property_t *p = utf8proc_get_property(c);
|
||||
printf("U+%s:\n"
|
||||
" category = %s\n"
|
||||
" combining_class = %d\n"
|
||||
" bidi_class = %d\n"
|
||||
" decomp_type = %d\n"
|
||||
" uppercase_mapping = %x\n"
|
||||
" lowercase_mapping = %x\n"
|
||||
" titlecase_mapping = %x\n"
|
||||
" comb_index = %d\n"
|
||||
" bidi_mirrored = %d\n"
|
||||
" comp_exclusion = %d\n"
|
||||
" ignorable = %d\n"
|
||||
" control_boundary = %d\n"
|
||||
" boundclass = %d\n"
|
||||
" charwidth = %d\n",
|
||||
argv[i],
|
||||
utf8proc_category_string(c),
|
||||
p->combining_class,
|
||||
p->bidi_class,
|
||||
p->decomp_type,
|
||||
utf8proc_toupper(c),
|
||||
utf8proc_tolower(c),
|
||||
utf8proc_totitle(c),
|
||||
p->comb_index,
|
||||
p->bidi_mirrored,
|
||||
p->comp_exclusion,
|
||||
p->ignorable,
|
||||
p->control_boundary,
|
||||
p->boundclass,
|
||||
utf8proc_charwidth(c));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
/* Common functions for our test programs. */
|
||||
|
||||
#include "tests.h"
|
||||
|
||||
size_t lineno = 0;
|
||||
|
||||
void check(int cond, const char *format, ...)
|
||||
{
|
||||
if (!cond) {
|
||||
va_list args;
|
||||
fprintf(stderr, "line %zd: ", lineno);
|
||||
va_start(args, format);
|
||||
vfprintf(stderr, format, args);
|
||||
va_end(args);
|
||||
fprintf(stderr, "\n");
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
size_t skipspaces(const char *buf, size_t i)
|
||||
{
|
||||
while (isspace(buf[i])) ++i;
|
||||
return i;
|
||||
}
|
||||
|
||||
/* if buf points to a sequence of codepoints encoded as hexadecimal strings,
|
||||
separated by whitespace, and terminated by any character not in
|
||||
[0-9a-fA-F] or whitespace, then stores the corresponding utf8 string
|
||||
in dest, returning the number of bytes read from buf */
|
||||
size_t encode(char *dest, const char *buf)
|
||||
{
|
||||
size_t i = 0, j, d = 0;
|
||||
for (;;) {
|
||||
int c;
|
||||
i = skipspaces(buf, i);
|
||||
for (j=i; buf[j] && strchr("0123456789abcdef", tolower(buf[j])); ++j)
|
||||
; /* find end of hex input */
|
||||
if (j == i) { /* no codepoint found */
|
||||
dest[d] = 0; /* NUL-terminate destination string */
|
||||
return i + 1;
|
||||
}
|
||||
check(sscanf(buf + i, "%x", (unsigned int *)&c) == 1, "invalid hex input %s", buf+i);
|
||||
i = j; /* skip to char after hex input */
|
||||
d += utf8proc_encode_char(c, (utf8proc_uint8_t *) (dest + d));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
/* Common functions and includes for our test programs. */
|
||||
|
||||
/*
|
||||
* Set feature macro to enable getline() and wcwidth().
|
||||
*
|
||||
* Please refer to section 2.2.1 of POSIX.1-2008:
|
||||
* http://pubs.opengroup.org/onlinepubs/9699919799/functions/V2_chap02.html#tag_15_02_01_02
|
||||
*/
|
||||
#define _XOPEN_SOURCE 700
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <ctype.h>
|
||||
#include <string.h>
|
||||
#include <stdarg.h>
|
||||
|
||||
#include "../utf8proc.h"
|
||||
|
||||
extern size_t lineno;
|
||||
|
||||
void check(int cond, const char *format, ...);
|
||||
size_t skipspaces(const char *buf, size_t i);
|
||||
size_t encode(char *dest, const char *buf);
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
#include "tests.h"
|
||||
#include <ctype.h>
|
||||
#include <wchar.h>
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
int c, error = 0;
|
||||
|
||||
(void) argc; /* unused */
|
||||
(void) argv; /* unused */
|
||||
|
||||
/* some simple sanity tests of */
|
||||
for (c = 0; c < 0xd800; c++) {
|
||||
if (!utf8proc_codepoint_valid(c)) {
|
||||
fprintf(stderr, "Failed: codepoint_valid(%04x) -> false\n", c);
|
||||
error++;
|
||||
}
|
||||
}
|
||||
for (;c < 0xe000; c++) {
|
||||
if (utf8proc_codepoint_valid(c)) {
|
||||
fprintf(stderr, "Failed: codepoint_valid(%04x) -> true\n", c);
|
||||
error++;
|
||||
}
|
||||
}
|
||||
for (;c < 0x110000; c++) {
|
||||
if (!utf8proc_codepoint_valid(c)) {
|
||||
fprintf(stderr, "Failed: codepoint_valid(%06x) -> false\n", c);
|
||||
error++;
|
||||
}
|
||||
}
|
||||
for (;c < 0x110010; c++) {
|
||||
if (utf8proc_codepoint_valid(c)) {
|
||||
fprintf(stderr, "Failed: codepoint_valid(%06x) -> true\n", c);
|
||||
error++;
|
||||
}
|
||||
}
|
||||
check(!error, "utf8proc_codepoint_valid FAILED %d tests.", error);
|
||||
printf("Validity tests SUCCEEDED.\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -0,0 +1,755 @@
|
|||
/* -*- mode: c; c-basic-offset: 2; tab-width: 2; indent-tabs-mode: nil -*- */
|
||||
/*
|
||||
* Copyright (c) 2015 Steven G. Johnson, Jiahao Chen, Peter Colberg, Tony Kelman, Scott P. Jones, and other contributors.
|
||||
* Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
* DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
/*
|
||||
* This library contains derived data from a modified version of the
|
||||
* Unicode data files.
|
||||
*
|
||||
* The original data files are available at
|
||||
* http://www.unicode.org/Public/UNIDATA/
|
||||
*
|
||||
* Please notice the copyright statement in the file "utf8proc_data.c".
|
||||
*/
|
||||
|
||||
|
||||
/*
|
||||
* File name: utf8proc.c
|
||||
*
|
||||
* Description:
|
||||
* Implementation of libutf8proc.
|
||||
*/
|
||||
|
||||
|
||||
#include "utf8proc.h"
|
||||
#include "utf8proc_data.c"
|
||||
|
||||
|
||||
UTF8PROC_DLLEXPORT const utf8proc_int8_t utf8proc_utf8class[256] = {
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||||
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
||||
4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0 };
|
||||
|
||||
#define UTF8PROC_HANGUL_SBASE 0xAC00
|
||||
#define UTF8PROC_HANGUL_LBASE 0x1100
|
||||
#define UTF8PROC_HANGUL_VBASE 0x1161
|
||||
#define UTF8PROC_HANGUL_TBASE 0x11A7
|
||||
#define UTF8PROC_HANGUL_LCOUNT 19
|
||||
#define UTF8PROC_HANGUL_VCOUNT 21
|
||||
#define UTF8PROC_HANGUL_TCOUNT 28
|
||||
#define UTF8PROC_HANGUL_NCOUNT 588
|
||||
#define UTF8PROC_HANGUL_SCOUNT 11172
|
||||
/* END is exclusive */
|
||||
#define UTF8PROC_HANGUL_L_START 0x1100
|
||||
#define UTF8PROC_HANGUL_L_END 0x115A
|
||||
#define UTF8PROC_HANGUL_L_FILLER 0x115F
|
||||
#define UTF8PROC_HANGUL_V_START 0x1160
|
||||
#define UTF8PROC_HANGUL_V_END 0x11A3
|
||||
#define UTF8PROC_HANGUL_T_START 0x11A8
|
||||
#define UTF8PROC_HANGUL_T_END 0x11FA
|
||||
#define UTF8PROC_HANGUL_S_START 0xAC00
|
||||
#define UTF8PROC_HANGUL_S_END 0xD7A4
|
||||
|
||||
/* Should follow semantic-versioning rules (semver.org) based on API
|
||||
compatibility. (Note that the shared-library version number will
|
||||
be different, being based on ABI compatibility.): */
|
||||
#define STRINGIZEx(x) #x
|
||||
#define STRINGIZE(x) STRINGIZEx(x)
|
||||
UTF8PROC_DLLEXPORT const char *utf8proc_version(void) {
|
||||
return STRINGIZE(UTF8PROC_VERSION_MAJOR) "." STRINGIZE(UTF8PROC_VERSION_MINOR) "." STRINGIZE(UTF8PROC_VERSION_PATCH) "";
|
||||
}
|
||||
|
||||
UTF8PROC_DLLEXPORT const char *utf8proc_errmsg(utf8proc_ssize_t errcode) {
|
||||
switch (errcode) {
|
||||
case UTF8PROC_ERROR_NOMEM:
|
||||
return "Memory for processing UTF-8 data could not be allocated.";
|
||||
case UTF8PROC_ERROR_OVERFLOW:
|
||||
return "UTF-8 string is too long to be processed.";
|
||||
case UTF8PROC_ERROR_INVALIDUTF8:
|
||||
return "Invalid UTF-8 string";
|
||||
case UTF8PROC_ERROR_NOTASSIGNED:
|
||||
return "Unassigned Unicode code point found in UTF-8 string.";
|
||||
case UTF8PROC_ERROR_INVALIDOPTS:
|
||||
return "Invalid options for UTF-8 processing chosen.";
|
||||
default:
|
||||
return "An unknown error occurred while processing UTF-8 data.";
|
||||
}
|
||||
}
|
||||
|
||||
#define utf_cont(ch) (((ch) & 0xc0) == 0x80)
|
||||
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_iterate(
|
||||
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *dst
|
||||
) {
|
||||
utf8proc_uint32_t uc;
|
||||
const utf8proc_uint8_t *end;
|
||||
|
||||
*dst = -1;
|
||||
if (!strlen) return 0;
|
||||
end = str + ((strlen < 0) ? 4 : strlen);
|
||||
uc = *str++;
|
||||
if (uc < 0x80) {
|
||||
*dst = uc;
|
||||
return 1;
|
||||
}
|
||||
// Must be between 0xc2 and 0xf4 inclusive to be valid
|
||||
if ((uc - 0xc2) > (0xf4-0xc2)) return UTF8PROC_ERROR_INVALIDUTF8;
|
||||
if (uc < 0xe0) { // 2-byte sequence
|
||||
// Must have valid continuation character
|
||||
if (str >= end || !utf_cont(*str)) return UTF8PROC_ERROR_INVALIDUTF8;
|
||||
*dst = ((uc & 0x1f)<<6) | (*str & 0x3f);
|
||||
return 2;
|
||||
}
|
||||
if (uc < 0xf0) { // 3-byte sequence
|
||||
if ((str + 1 >= end) || !utf_cont(*str) || !utf_cont(str[1]))
|
||||
return UTF8PROC_ERROR_INVALIDUTF8;
|
||||
// Check for surrogate chars
|
||||
if (uc == 0xed && *str > 0x9f)
|
||||
return UTF8PROC_ERROR_INVALIDUTF8;
|
||||
uc = ((uc & 0xf)<<12) | ((*str & 0x3f)<<6) | (str[1] & 0x3f);
|
||||
if (uc < 0x800)
|
||||
return UTF8PROC_ERROR_INVALIDUTF8;
|
||||
*dst = uc;
|
||||
return 3;
|
||||
}
|
||||
// 4-byte sequence
|
||||
// Must have 3 valid continuation characters
|
||||
if ((str + 2 >= end) || !utf_cont(*str) || !utf_cont(str[1]) || !utf_cont(str[2]))
|
||||
return UTF8PROC_ERROR_INVALIDUTF8;
|
||||
// Make sure in correct range (0x10000 - 0x10ffff)
|
||||
if (uc == 0xf0) {
|
||||
if (*str < 0x90) return UTF8PROC_ERROR_INVALIDUTF8;
|
||||
} else if (uc == 0xf4) {
|
||||
if (*str > 0x8f) return UTF8PROC_ERROR_INVALIDUTF8;
|
||||
}
|
||||
*dst = ((uc & 7)<<18) | ((*str & 0x3f)<<12) | ((str[1] & 0x3f)<<6) | (str[2] & 0x3f);
|
||||
return 4;
|
||||
}
|
||||
|
||||
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_codepoint_valid(utf8proc_int32_t uc) {
|
||||
return (((utf8proc_uint32_t)uc)-0xd800 > 0x07ff) && ((utf8proc_uint32_t)uc < 0x110000);
|
||||
}
|
||||
|
||||
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst) {
|
||||
if (uc < 0x00) {
|
||||
return 0;
|
||||
} else if (uc < 0x80) {
|
||||
dst[0] = (utf8proc_uint8_t) uc;
|
||||
return 1;
|
||||
} else if (uc < 0x800) {
|
||||
dst[0] = (utf8proc_uint8_t)(0xC0 + (uc >> 6));
|
||||
dst[1] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
|
||||
return 2;
|
||||
// Note: we allow encoding 0xd800-0xdfff here, so as not to change
|
||||
// the API, however, these are actually invalid in UTF-8
|
||||
} else if (uc < 0x10000) {
|
||||
dst[0] = (utf8proc_uint8_t)(0xE0 + (uc >> 12));
|
||||
dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
|
||||
dst[2] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
|
||||
return 3;
|
||||
} else if (uc < 0x110000) {
|
||||
dst[0] = (utf8proc_uint8_t)(0xF0 + (uc >> 18));
|
||||
dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 12) & 0x3F));
|
||||
dst[2] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
|
||||
dst[3] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
|
||||
return 4;
|
||||
} else return 0;
|
||||
}
|
||||
|
||||
/* internal "unsafe" version that does not check whether uc is in range */
|
||||
static utf8proc_ssize_t unsafe_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst) {
|
||||
if (uc < 0x00) {
|
||||
return 0;
|
||||
} else if (uc < 0x80) {
|
||||
dst[0] = (utf8proc_uint8_t)uc;
|
||||
return 1;
|
||||
} else if (uc < 0x800) {
|
||||
dst[0] = (utf8proc_uint8_t)(0xC0 + (uc >> 6));
|
||||
dst[1] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
|
||||
return 2;
|
||||
} else if (uc == 0xFFFF) {
|
||||
dst[0] = (utf8proc_uint8_t)0xFF;
|
||||
return 1;
|
||||
} else if (uc == 0xFFFE) {
|
||||
dst[0] = (utf8proc_uint8_t)0xFE;
|
||||
return 1;
|
||||
} else if (uc < 0x10000) {
|
||||
dst[0] = (utf8proc_uint8_t)(0xE0 + (uc >> 12));
|
||||
dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
|
||||
dst[2] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
|
||||
return 3;
|
||||
} else if (uc < 0x110000) {
|
||||
dst[0] = (utf8proc_uint8_t)(0xF0 + (uc >> 18));
|
||||
dst[1] = (utf8proc_uint8_t)(0x80 + ((uc >> 12) & 0x3F));
|
||||
dst[2] = (utf8proc_uint8_t)(0x80 + ((uc >> 6) & 0x3F));
|
||||
dst[3] = (utf8proc_uint8_t)(0x80 + (uc & 0x3F));
|
||||
return 4;
|
||||
} else return 0;
|
||||
}
|
||||
|
||||
/* internal "unsafe" version that does not check whether uc is in range */
|
||||
static const utf8proc_property_t *unsafe_get_property(utf8proc_int32_t uc) {
|
||||
/* ASSERT: uc >= 0 && uc < 0x110000 */
|
||||
return utf8proc_properties + (
|
||||
utf8proc_stage2table[
|
||||
utf8proc_stage1table[uc >> 8] + (uc & 0xFF)
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
UTF8PROC_DLLEXPORT const utf8proc_property_t *utf8proc_get_property(utf8proc_int32_t uc) {
|
||||
return uc < 0 || uc >= 0x110000 ? utf8proc_properties : unsafe_get_property(uc);
|
||||
}
|
||||
|
||||
/* return whether there is a grapheme break between boundclasses lbc and tbc
|
||||
(according to the definition of extended grapheme clusters)
|
||||
|
||||
Rule numbering refers to TR29 Version 29 (Unicode 9.0.0):
|
||||
http://www.unicode.org/reports/tr29/tr29-29.html
|
||||
|
||||
CAVEATS:
|
||||
Please note that evaluation of GB10 (grapheme breaks between emoji zwj sequences)
|
||||
and GB 12/13 (regional indicator code points) require knowledge of previous characters
|
||||
and are thus not handled by this function. This may result in an incorrect break before
|
||||
an E_Modifier class codepoint and an incorrectly missing break between two
|
||||
REGIONAL_INDICATOR class code points if such support does not exist in the caller.
|
||||
|
||||
See the special support in grapheme_break_extended, for required bookkeeping by the caller.
|
||||
*/
|
||||
static utf8proc_bool grapheme_break_simple(int lbc, int tbc) {
|
||||
return
|
||||
(lbc == UTF8PROC_BOUNDCLASS_START) ? true : // GB1
|
||||
(lbc == UTF8PROC_BOUNDCLASS_CR && // GB3
|
||||
tbc == UTF8PROC_BOUNDCLASS_LF) ? false : // ---
|
||||
(lbc >= UTF8PROC_BOUNDCLASS_CR && lbc <= UTF8PROC_BOUNDCLASS_CONTROL) ? true : // GB4
|
||||
(tbc >= UTF8PROC_BOUNDCLASS_CR && tbc <= UTF8PROC_BOUNDCLASS_CONTROL) ? true : // GB5
|
||||
(lbc == UTF8PROC_BOUNDCLASS_L && // GB6
|
||||
(tbc == UTF8PROC_BOUNDCLASS_L || // ---
|
||||
tbc == UTF8PROC_BOUNDCLASS_V || // ---
|
||||
tbc == UTF8PROC_BOUNDCLASS_LV || // ---
|
||||
tbc == UTF8PROC_BOUNDCLASS_LVT)) ? false : // ---
|
||||
((lbc == UTF8PROC_BOUNDCLASS_LV || // GB7
|
||||
lbc == UTF8PROC_BOUNDCLASS_V) && // ---
|
||||
(tbc == UTF8PROC_BOUNDCLASS_V || // ---
|
||||
tbc == UTF8PROC_BOUNDCLASS_T)) ? false : // ---
|
||||
((lbc == UTF8PROC_BOUNDCLASS_LVT || // GB8
|
||||
lbc == UTF8PROC_BOUNDCLASS_T) && // ---
|
||||
tbc == UTF8PROC_BOUNDCLASS_T) ? false : // ---
|
||||
(tbc == UTF8PROC_BOUNDCLASS_EXTEND || // GB9
|
||||
tbc == UTF8PROC_BOUNDCLASS_ZWJ || // ---
|
||||
tbc == UTF8PROC_BOUNDCLASS_SPACINGMARK || // GB9a
|
||||
lbc == UTF8PROC_BOUNDCLASS_PREPEND) ? false : // GB9b
|
||||
((lbc == UTF8PROC_BOUNDCLASS_E_BASE || // GB10 (requires additional handling below)
|
||||
lbc == UTF8PROC_BOUNDCLASS_E_BASE_GAZ) && // ----
|
||||
tbc == UTF8PROC_BOUNDCLASS_E_MODIFIER) ? false : // ----
|
||||
(lbc == UTF8PROC_BOUNDCLASS_ZWJ && // GB11
|
||||
(tbc == UTF8PROC_BOUNDCLASS_GLUE_AFTER_ZWJ || // ----
|
||||
tbc == UTF8PROC_BOUNDCLASS_E_BASE_GAZ)) ? false : // ----
|
||||
(lbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR && // GB12/13 (requires additional handling below)
|
||||
tbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR) ? false : // ----
|
||||
true; // GB999
|
||||
}
|
||||
|
||||
static utf8proc_bool grapheme_break_extended(int lbc, int tbc, utf8proc_int32_t *state)
|
||||
{
|
||||
int lbc_override = lbc;
|
||||
if (state && *state != UTF8PROC_BOUNDCLASS_START)
|
||||
lbc_override = *state;
|
||||
utf8proc_bool break_permitted = grapheme_break_simple(lbc_override, tbc);
|
||||
if (state) {
|
||||
// Special support for GB 12/13 made possible by GB999. After two RI
|
||||
// class codepoints we want to force a break. Do this by resetting the
|
||||
// second RI's bound class to UTF8PROC_BOUNDCLASS_OTHER, to force a break
|
||||
// after that character according to GB999 (unless of course such a break is
|
||||
// forbidden by a different rule such as GB9).
|
||||
if (*state == tbc && tbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR)
|
||||
*state = UTF8PROC_BOUNDCLASS_OTHER;
|
||||
// Special support for GB10. Fold any EXTEND codepoints into the previous
|
||||
// boundclass if we're dealing with an emoji base boundclass.
|
||||
else if ((*state == UTF8PROC_BOUNDCLASS_E_BASE ||
|
||||
*state == UTF8PROC_BOUNDCLASS_E_BASE_GAZ) &&
|
||||
tbc == UTF8PROC_BOUNDCLASS_EXTEND)
|
||||
*state = UTF8PROC_BOUNDCLASS_E_BASE;
|
||||
else
|
||||
*state = tbc;
|
||||
}
|
||||
return break_permitted;
|
||||
}
|
||||
|
||||
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break_stateful(
|
||||
utf8proc_int32_t c1, utf8proc_int32_t c2, utf8proc_int32_t *state) {
|
||||
|
||||
return grapheme_break_extended(utf8proc_get_property(c1)->boundclass,
|
||||
utf8proc_get_property(c2)->boundclass,
|
||||
state);
|
||||
}
|
||||
|
||||
|
||||
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break(
|
||||
utf8proc_int32_t c1, utf8proc_int32_t c2) {
|
||||
return utf8proc_grapheme_break_stateful(c1, c2, NULL);
|
||||
}
|
||||
|
||||
static utf8proc_int32_t seqindex_decode_entry(const utf8proc_uint16_t **entry)
|
||||
{
|
||||
utf8proc_int32_t entry_cp = **entry;
|
||||
if ((entry_cp & 0xF800) == 0xD800) {
|
||||
*entry = *entry + 1;
|
||||
entry_cp = ((entry_cp & 0x03FF) << 10) | (**entry & 0x03FF);
|
||||
entry_cp += 0x10000;
|
||||
}
|
||||
return entry_cp;
|
||||
}
|
||||
|
||||
static utf8proc_int32_t seqindex_decode_index(const utf8proc_uint32_t seqindex)
|
||||
{
|
||||
const utf8proc_uint16_t *entry = &utf8proc_sequences[seqindex];
|
||||
return seqindex_decode_entry(&entry);
|
||||
}
|
||||
|
||||
static utf8proc_ssize_t seqindex_write_char_decomposed(utf8proc_uint16_t seqindex, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize, utf8proc_option_t options, int *last_boundclass) {
|
||||
utf8proc_ssize_t written = 0;
|
||||
const utf8proc_uint16_t *entry = &utf8proc_sequences[seqindex & 0x1FFF];
|
||||
int len = seqindex >> 13;
|
||||
if (len >= 7) {
|
||||
len = *entry;
|
||||
entry++;
|
||||
}
|
||||
for (; len >= 0; entry++, len--) {
|
||||
utf8proc_int32_t entry_cp = seqindex_decode_entry(&entry);
|
||||
|
||||
written += utf8proc_decompose_char(entry_cp, dst+written,
|
||||
(bufsize > written) ? (bufsize - written) : 0, options,
|
||||
last_boundclass);
|
||||
if (written < 0) return UTF8PROC_ERROR_OVERFLOW;
|
||||
}
|
||||
return written;
|
||||
}
|
||||
|
||||
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_tolower(utf8proc_int32_t c)
|
||||
{
|
||||
utf8proc_int32_t cl = utf8proc_get_property(c)->lowercase_seqindex;
|
||||
return cl != UINT16_MAX ? seqindex_decode_index(cl) : c;
|
||||
}
|
||||
|
||||
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_toupper(utf8proc_int32_t c)
|
||||
{
|
||||
utf8proc_int32_t cu = utf8proc_get_property(c)->uppercase_seqindex;
|
||||
return cu != UINT16_MAX ? seqindex_decode_index(cu) : c;
|
||||
}
|
||||
|
||||
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_totitle(utf8proc_int32_t c)
|
||||
{
|
||||
utf8proc_int32_t cu = utf8proc_get_property(c)->titlecase_seqindex;
|
||||
return cu != UINT16_MAX ? seqindex_decode_index(cu) : c;
|
||||
}
|
||||
|
||||
/* return a character width analogous to wcwidth (except portable and
|
||||
hopefully less buggy than most system wcwidth functions). */
|
||||
UTF8PROC_DLLEXPORT int utf8proc_charwidth(utf8proc_int32_t c) {
|
||||
return utf8proc_get_property(c)->charwidth;
|
||||
}
|
||||
|
||||
UTF8PROC_DLLEXPORT utf8proc_category_t utf8proc_category(utf8proc_int32_t c) {
|
||||
return utf8proc_get_property(c)->category;
|
||||
}
|
||||
|
||||
UTF8PROC_DLLEXPORT const char *utf8proc_category_string(utf8proc_int32_t c) {
|
||||
static const char s[][3] = {"Cn","Lu","Ll","Lt","Lm","Lo","Mn","Mc","Me","Nd","Nl","No","Pc","Pd","Ps","Pe","Pi","Pf","Po","Sm","Sc","Sk","So","Zs","Zl","Zp","Cc","Cf","Cs","Co"};
|
||||
return s[utf8proc_category(c)];
|
||||
}
|
||||
|
||||
#define utf8proc_decompose_lump(replacement_uc) \
|
||||
return utf8proc_decompose_char((replacement_uc), dst, bufsize, \
|
||||
options & ~UTF8PROC_LUMP, last_boundclass)
|
||||
|
||||
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char(utf8proc_int32_t uc, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize, utf8proc_option_t options, int *last_boundclass) {
|
||||
const utf8proc_property_t *property;
|
||||
utf8proc_propval_t category;
|
||||
utf8proc_int32_t hangul_sindex;
|
||||
if (uc < 0 || uc >= 0x110000) return UTF8PROC_ERROR_NOTASSIGNED;
|
||||
property = unsafe_get_property(uc);
|
||||
category = property->category;
|
||||
hangul_sindex = uc - UTF8PROC_HANGUL_SBASE;
|
||||
if (options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) {
|
||||
if (hangul_sindex >= 0 && hangul_sindex < UTF8PROC_HANGUL_SCOUNT) {
|
||||
utf8proc_int32_t hangul_tindex;
|
||||
if (bufsize >= 1) {
|
||||
dst[0] = UTF8PROC_HANGUL_LBASE +
|
||||
hangul_sindex / UTF8PROC_HANGUL_NCOUNT;
|
||||
if (bufsize >= 2) dst[1] = UTF8PROC_HANGUL_VBASE +
|
||||
(hangul_sindex % UTF8PROC_HANGUL_NCOUNT) / UTF8PROC_HANGUL_TCOUNT;
|
||||
}
|
||||
hangul_tindex = hangul_sindex % UTF8PROC_HANGUL_TCOUNT;
|
||||
if (!hangul_tindex) return 2;
|
||||
if (bufsize >= 3) dst[2] = UTF8PROC_HANGUL_TBASE + hangul_tindex;
|
||||
return 3;
|
||||
}
|
||||
}
|
||||
if (options & UTF8PROC_REJECTNA) {
|
||||
if (!category) return UTF8PROC_ERROR_NOTASSIGNED;
|
||||
}
|
||||
if (options & UTF8PROC_IGNORE) {
|
||||
if (property->ignorable) return 0;
|
||||
}
|
||||
if (options & UTF8PROC_LUMP) {
|
||||
if (category == UTF8PROC_CATEGORY_ZS) utf8proc_decompose_lump(0x0020);
|
||||
if (uc == 0x2018 || uc == 0x2019 || uc == 0x02BC || uc == 0x02C8)
|
||||
utf8proc_decompose_lump(0x0027);
|
||||
if (category == UTF8PROC_CATEGORY_PD || uc == 0x2212)
|
||||
utf8proc_decompose_lump(0x002D);
|
||||
if (uc == 0x2044 || uc == 0x2215) utf8proc_decompose_lump(0x002F);
|
||||
if (uc == 0x2236) utf8proc_decompose_lump(0x003A);
|
||||
if (uc == 0x2039 || uc == 0x2329 || uc == 0x3008)
|
||||
utf8proc_decompose_lump(0x003C);
|
||||
if (uc == 0x203A || uc == 0x232A || uc == 0x3009)
|
||||
utf8proc_decompose_lump(0x003E);
|
||||
if (uc == 0x2216) utf8proc_decompose_lump(0x005C);
|
||||
if (uc == 0x02C4 || uc == 0x02C6 || uc == 0x2038 || uc == 0x2303)
|
||||
utf8proc_decompose_lump(0x005E);
|
||||
if (category == UTF8PROC_CATEGORY_PC || uc == 0x02CD)
|
||||
utf8proc_decompose_lump(0x005F);
|
||||
if (uc == 0x02CB) utf8proc_decompose_lump(0x0060);
|
||||
if (uc == 0x2223) utf8proc_decompose_lump(0x007C);
|
||||
if (uc == 0x223C) utf8proc_decompose_lump(0x007E);
|
||||
if ((options & UTF8PROC_NLF2LS) && (options & UTF8PROC_NLF2PS)) {
|
||||
if (category == UTF8PROC_CATEGORY_ZL ||
|
||||
category == UTF8PROC_CATEGORY_ZP)
|
||||
utf8proc_decompose_lump(0x000A);
|
||||
}
|
||||
}
|
||||
if (options & UTF8PROC_STRIPMARK) {
|
||||
if (category == UTF8PROC_CATEGORY_MN ||
|
||||
category == UTF8PROC_CATEGORY_MC ||
|
||||
category == UTF8PROC_CATEGORY_ME) return 0;
|
||||
}
|
||||
if (options & UTF8PROC_CASEFOLD) {
|
||||
if (property->casefold_seqindex != UINT16_MAX) {
|
||||
return seqindex_write_char_decomposed(property->casefold_seqindex, dst, bufsize, options, last_boundclass);
|
||||
}
|
||||
}
|
||||
if (options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) {
|
||||
if (property->decomp_seqindex != UINT16_MAX &&
|
||||
(!property->decomp_type || (options & UTF8PROC_COMPAT))) {
|
||||
return seqindex_write_char_decomposed(property->decomp_seqindex, dst, bufsize, options, last_boundclass);
|
||||
}
|
||||
}
|
||||
if (options & UTF8PROC_CHARBOUND) {
|
||||
utf8proc_bool boundary;
|
||||
int tbc = property->boundclass;
|
||||
boundary = grapheme_break_extended(*last_boundclass, tbc, last_boundclass);
|
||||
if (boundary) {
|
||||
if (bufsize >= 1) dst[0] = 0xFFFF;
|
||||
if (bufsize >= 2) dst[1] = uc;
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
if (bufsize >= 1) *dst = uc;
|
||||
return 1;
|
||||
}
|
||||
|
||||
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose(
|
||||
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
|
||||
utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options
|
||||
) {
|
||||
return utf8proc_decompose_custom(str, strlen, buffer, bufsize, options, NULL, NULL);
|
||||
}
|
||||
|
||||
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_custom(
|
||||
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
|
||||
utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options,
|
||||
utf8proc_custom_func custom_func, void *custom_data
|
||||
) {
|
||||
/* strlen will be ignored, if UTF8PROC_NULLTERM is set in options */
|
||||
utf8proc_ssize_t wpos = 0;
|
||||
if ((options & UTF8PROC_COMPOSE) && (options & UTF8PROC_DECOMPOSE))
|
||||
return UTF8PROC_ERROR_INVALIDOPTS;
|
||||
if ((options & UTF8PROC_STRIPMARK) &&
|
||||
!(options & UTF8PROC_COMPOSE) && !(options & UTF8PROC_DECOMPOSE))
|
||||
return UTF8PROC_ERROR_INVALIDOPTS;
|
||||
{
|
||||
utf8proc_int32_t uc;
|
||||
utf8proc_ssize_t rpos = 0;
|
||||
utf8proc_ssize_t decomp_result;
|
||||
int boundclass = UTF8PROC_BOUNDCLASS_START;
|
||||
while (1) {
|
||||
if (options & UTF8PROC_NULLTERM) {
|
||||
rpos += utf8proc_iterate(str + rpos, -1, &uc);
|
||||
/* checking of return value is not necessary,
|
||||
as 'uc' is < 0 in case of error */
|
||||
if (uc < 0) return UTF8PROC_ERROR_INVALIDUTF8;
|
||||
if (rpos < 0) return UTF8PROC_ERROR_OVERFLOW;
|
||||
if (uc == 0) break;
|
||||
} else {
|
||||
if (rpos >= strlen) break;
|
||||
rpos += utf8proc_iterate(str + rpos, strlen - rpos, &uc);
|
||||
if (uc < 0) return UTF8PROC_ERROR_INVALIDUTF8;
|
||||
}
|
||||
if (custom_func != NULL) {
|
||||
uc = custom_func(uc, custom_data); /* user-specified custom mapping */
|
||||
}
|
||||
decomp_result = utf8proc_decompose_char(
|
||||
uc, buffer + wpos, (bufsize > wpos) ? (bufsize - wpos) : 0, options,
|
||||
&boundclass
|
||||
);
|
||||
if (decomp_result < 0) return decomp_result;
|
||||
wpos += decomp_result;
|
||||
/* prohibiting integer overflows due to too long strings: */
|
||||
if (wpos < 0 ||
|
||||
wpos > (utf8proc_ssize_t)(SSIZE_MAX/sizeof(utf8proc_int32_t)/2))
|
||||
return UTF8PROC_ERROR_OVERFLOW;
|
||||
}
|
||||
}
|
||||
if ((options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) && bufsize >= wpos) {
|
||||
utf8proc_ssize_t pos = 0;
|
||||
while (pos < wpos-1) {
|
||||
utf8proc_int32_t uc1, uc2;
|
||||
const utf8proc_property_t *property1, *property2;
|
||||
uc1 = buffer[pos];
|
||||
uc2 = buffer[pos+1];
|
||||
property1 = unsafe_get_property(uc1);
|
||||
property2 = unsafe_get_property(uc2);
|
||||
if (property1->combining_class > property2->combining_class &&
|
||||
property2->combining_class > 0) {
|
||||
buffer[pos] = uc2;
|
||||
buffer[pos+1] = uc1;
|
||||
if (pos > 0) pos--; else pos++;
|
||||
} else {
|
||||
pos++;
|
||||
}
|
||||
}
|
||||
}
|
||||
return wpos;
|
||||
}
|
||||
|
||||
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_normalize_utf32(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options) {
|
||||
/* UTF8PROC_NULLTERM option will be ignored, 'length' is never ignored */
|
||||
if (options & (UTF8PROC_NLF2LS | UTF8PROC_NLF2PS | UTF8PROC_STRIPCC)) {
|
||||
utf8proc_ssize_t rpos;
|
||||
utf8proc_ssize_t wpos = 0;
|
||||
utf8proc_int32_t uc;
|
||||
for (rpos = 0; rpos < length; rpos++) {
|
||||
uc = buffer[rpos];
|
||||
if (uc == 0x000D && rpos < length-1 && buffer[rpos+1] == 0x000A) rpos++;
|
||||
if (uc == 0x000A || uc == 0x000D || uc == 0x0085 ||
|
||||
((options & UTF8PROC_STRIPCC) && (uc == 0x000B || uc == 0x000C))) {
|
||||
if (options & UTF8PROC_NLF2LS) {
|
||||
if (options & UTF8PROC_NLF2PS) {
|
||||
buffer[wpos++] = 0x000A;
|
||||
} else {
|
||||
buffer[wpos++] = 0x2028;
|
||||
}
|
||||
} else {
|
||||
if (options & UTF8PROC_NLF2PS) {
|
||||
buffer[wpos++] = 0x2029;
|
||||
} else {
|
||||
buffer[wpos++] = 0x0020;
|
||||
}
|
||||
}
|
||||
} else if ((options & UTF8PROC_STRIPCC) &&
|
||||
(uc < 0x0020 || (uc >= 0x007F && uc < 0x00A0))) {
|
||||
if (uc == 0x0009) buffer[wpos++] = 0x0020;
|
||||
} else {
|
||||
buffer[wpos++] = uc;
|
||||
}
|
||||
}
|
||||
length = wpos;
|
||||
}
|
||||
if (options & UTF8PROC_COMPOSE) {
|
||||
utf8proc_int32_t *starter = NULL;
|
||||
utf8proc_int32_t current_char;
|
||||
const utf8proc_property_t *starter_property = NULL, *current_property;
|
||||
utf8proc_propval_t max_combining_class = -1;
|
||||
utf8proc_ssize_t rpos;
|
||||
utf8proc_ssize_t wpos = 0;
|
||||
utf8proc_int32_t composition;
|
||||
for (rpos = 0; rpos < length; rpos++) {
|
||||
current_char = buffer[rpos];
|
||||
current_property = unsafe_get_property(current_char);
|
||||
if (starter && current_property->combining_class > max_combining_class) {
|
||||
/* combination perhaps possible */
|
||||
utf8proc_int32_t hangul_lindex;
|
||||
utf8proc_int32_t hangul_sindex;
|
||||
hangul_lindex = *starter - UTF8PROC_HANGUL_LBASE;
|
||||
if (hangul_lindex >= 0 && hangul_lindex < UTF8PROC_HANGUL_LCOUNT) {
|
||||
utf8proc_int32_t hangul_vindex;
|
||||
hangul_vindex = current_char - UTF8PROC_HANGUL_VBASE;
|
||||
if (hangul_vindex >= 0 && hangul_vindex < UTF8PROC_HANGUL_VCOUNT) {
|
||||
*starter = UTF8PROC_HANGUL_SBASE +
|
||||
(hangul_lindex * UTF8PROC_HANGUL_VCOUNT + hangul_vindex) *
|
||||
UTF8PROC_HANGUL_TCOUNT;
|
||||
starter_property = NULL;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
hangul_sindex = *starter - UTF8PROC_HANGUL_SBASE;
|
||||
if (hangul_sindex >= 0 && hangul_sindex < UTF8PROC_HANGUL_SCOUNT &&
|
||||
(hangul_sindex % UTF8PROC_HANGUL_TCOUNT) == 0) {
|
||||
utf8proc_int32_t hangul_tindex;
|
||||
hangul_tindex = current_char - UTF8PROC_HANGUL_TBASE;
|
||||
if (hangul_tindex >= 0 && hangul_tindex < UTF8PROC_HANGUL_TCOUNT) {
|
||||
*starter += hangul_tindex;
|
||||
starter_property = NULL;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (!starter_property) {
|
||||
starter_property = unsafe_get_property(*starter);
|
||||
}
|
||||
if (starter_property->comb_index < 0x8000 &&
|
||||
current_property->comb_index != UINT16_MAX &&
|
||||
current_property->comb_index >= 0x8000) {
|
||||
int sidx = starter_property->comb_index;
|
||||
int idx = (current_property->comb_index & 0x3FFF) - utf8proc_combinations[sidx];
|
||||
if (idx >= 0 && idx <= utf8proc_combinations[sidx + 1] ) {
|
||||
idx += sidx + 2;
|
||||
if (current_property->comb_index & 0x4000) {
|
||||
composition = (utf8proc_combinations[idx] << 16) | utf8proc_combinations[idx+1];
|
||||
} else
|
||||
composition = utf8proc_combinations[idx];
|
||||
|
||||
if (composition > 0 && (!(options & UTF8PROC_STABLE) ||
|
||||
!(unsafe_get_property(composition)->comp_exclusion))) {
|
||||
*starter = composition;
|
||||
starter_property = NULL;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
buffer[wpos] = current_char;
|
||||
if (current_property->combining_class) {
|
||||
if (current_property->combining_class > max_combining_class) {
|
||||
max_combining_class = current_property->combining_class;
|
||||
}
|
||||
} else {
|
||||
starter = buffer + wpos;
|
||||
starter_property = NULL;
|
||||
max_combining_class = -1;
|
||||
}
|
||||
wpos++;
|
||||
}
|
||||
length = wpos;
|
||||
}
|
||||
return length;
|
||||
}
|
||||
|
||||
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options) {
|
||||
/* UTF8PROC_NULLTERM option will be ignored, 'length' is never ignored
|
||||
ASSERT: 'buffer' has one spare byte of free space at the end! */
|
||||
length = utf8proc_normalize_utf32(buffer, length, options);
|
||||
if (length < 0) return length;
|
||||
{
|
||||
utf8proc_ssize_t rpos, wpos = 0;
|
||||
utf8proc_int32_t uc;
|
||||
if (options & UTF8PROC_CHARBOUND) {
|
||||
for (rpos = 0; rpos < length; rpos++) {
|
||||
uc = buffer[rpos];
|
||||
wpos += unsafe_encode_char(uc, ((utf8proc_uint8_t *)buffer) + wpos);
|
||||
}
|
||||
} else {
|
||||
for (rpos = 0; rpos < length; rpos++) {
|
||||
uc = buffer[rpos];
|
||||
wpos += utf8proc_encode_char(uc, ((utf8proc_uint8_t *)buffer) + wpos);
|
||||
}
|
||||
}
|
||||
((utf8proc_uint8_t *)buffer)[wpos] = 0;
|
||||
return wpos;
|
||||
}
|
||||
}
|
||||
|
||||
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map(
|
||||
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options
|
||||
) {
|
||||
return utf8proc_map_custom(str, strlen, dstptr, options, NULL, NULL);
|
||||
}
|
||||
|
||||
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map_custom(
|
||||
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options,
|
||||
utf8proc_custom_func custom_func, void *custom_data
|
||||
) {
|
||||
utf8proc_int32_t *buffer;
|
||||
utf8proc_ssize_t result;
|
||||
*dstptr = NULL;
|
||||
result = utf8proc_decompose_custom(str, strlen, NULL, 0, options, custom_func, custom_data);
|
||||
if (result < 0) return result;
|
||||
buffer = (utf8proc_int32_t *) malloc(result * sizeof(utf8proc_int32_t) + 1);
|
||||
if (!buffer) return UTF8PROC_ERROR_NOMEM;
|
||||
result = utf8proc_decompose_custom(str, strlen, buffer, result, options, custom_func, custom_data);
|
||||
if (result < 0) {
|
||||
free(buffer);
|
||||
return result;
|
||||
}
|
||||
result = utf8proc_reencode(buffer, result, options);
|
||||
if (result < 0) {
|
||||
free(buffer);
|
||||
return result;
|
||||
}
|
||||
{
|
||||
utf8proc_int32_t *newptr;
|
||||
newptr = (utf8proc_int32_t *) realloc(buffer, (size_t)result+1);
|
||||
if (newptr) buffer = newptr;
|
||||
}
|
||||
*dstptr = (utf8proc_uint8_t *)buffer;
|
||||
return result;
|
||||
}
|
||||
|
||||
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFD(const utf8proc_uint8_t *str) {
|
||||
utf8proc_uint8_t *retval;
|
||||
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
|
||||
UTF8PROC_DECOMPOSE);
|
||||
return retval;
|
||||
}
|
||||
|
||||
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFC(const utf8proc_uint8_t *str) {
|
||||
utf8proc_uint8_t *retval;
|
||||
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
|
||||
UTF8PROC_COMPOSE);
|
||||
return retval;
|
||||
}
|
||||
|
||||
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKD(const utf8proc_uint8_t *str) {
|
||||
utf8proc_uint8_t *retval;
|
||||
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
|
||||
UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT);
|
||||
return retval;
|
||||
}
|
||||
|
||||
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC(const utf8proc_uint8_t *str) {
|
||||
utf8proc_uint8_t *retval;
|
||||
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
|
||||
UTF8PROC_COMPOSE | UTF8PROC_COMPAT);
|
||||
return retval;
|
||||
}
|
||||
|
|
@ -0,0 +1,689 @@
|
|||
/*
|
||||
* Copyright (c) 2015 Steven G. Johnson, Jiahao Chen, Peter Colberg, Tony Kelman, Scott P. Jones, and other contributors.
|
||||
* Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
* DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
|
||||
/**
|
||||
* @mainpage
|
||||
*
|
||||
* utf8proc is a free/open-source (MIT/expat licensed) C library
|
||||
* providing Unicode normalization, case-folding, and other operations
|
||||
* for strings in the UTF-8 encoding, supporting Unicode version
|
||||
* 9.0.0. See the utf8proc home page (http://julialang.org/utf8proc/)
|
||||
* for downloads and other information, or the source code on github
|
||||
* (https://github.com/JuliaLang/utf8proc).
|
||||
*
|
||||
* For the utf8proc API documentation, see: @ref utf8proc.h
|
||||
*
|
||||
* The features of utf8proc include:
|
||||
*
|
||||
* - Transformation of strings (@ref utf8proc_map) to:
|
||||
* - decompose (@ref UTF8PROC_DECOMPOSE) or compose (@ref UTF8PROC_COMPOSE) Unicode combining characters (http://en.wikipedia.org/wiki/Combining_character)
|
||||
* - canonicalize Unicode compatibility characters (@ref UTF8PROC_COMPAT)
|
||||
* - strip "ignorable" (@ref UTF8PROC_IGNORE) characters, control characters (@ref UTF8PROC_STRIPCC), or combining characters such as accents (@ref UTF8PROC_STRIPMARK)
|
||||
* - case-folding (@ref UTF8PROC_CASEFOLD)
|
||||
* - Unicode normalization: @ref utf8proc_NFD, @ref utf8proc_NFC, @ref utf8proc_NFKD, @ref utf8proc_NFKC
|
||||
* - Detecting grapheme boundaries (@ref utf8proc_grapheme_break and @ref UTF8PROC_CHARBOUND)
|
||||
* - Character-width computation: @ref utf8proc_charwidth
|
||||
* - Classification of characters by Unicode category: @ref utf8proc_category and @ref utf8proc_category_string
|
||||
* - Encode (@ref utf8proc_encode_char) and decode (@ref utf8proc_iterate) Unicode codepoints to/from UTF-8.
|
||||
*/
|
||||
|
||||
/** @file */
|
||||
|
||||
#ifndef UTF8PROC_H
|
||||
#define UTF8PROC_H
|
||||
|
||||
/** @name API version
|
||||
*
|
||||
* The utf8proc API version MAJOR.MINOR.PATCH, following
|
||||
* semantic-versioning rules (http://semver.org) based on API
|
||||
* compatibility.
|
||||
*
|
||||
* This is also returned at runtime by @ref utf8proc_version; however, the
|
||||
* runtime version may append a string like "-dev" to the version number
|
||||
* for prerelease versions.
|
||||
*
|
||||
* @note The shared-library version number in the Makefile
|
||||
* (and CMakeLists.txt, and MANIFEST) may be different,
|
||||
* being based on ABI compatibility rather than API compatibility.
|
||||
*/
|
||||
/** @{ */
|
||||
/** The MAJOR version number (increased when backwards API compatibility is broken). */
|
||||
#define UTF8PROC_VERSION_MAJOR 2
|
||||
/** The MINOR version number (increased when new functionality is added in a backwards-compatible manner). */
|
||||
#define UTF8PROC_VERSION_MINOR 1
|
||||
/** The PATCH version (increased for fixes that do not change the API). */
|
||||
#define UTF8PROC_VERSION_PATCH 0
|
||||
/** @} */
|
||||
|
||||
#include <stdlib.h>
|
||||
|
||||
#if defined(_MSC_VER) && _MSC_VER < 1800
|
||||
// MSVC prior to 2013 lacked stdbool.h and inttypes.h
|
||||
typedef signed char utf8proc_int8_t;
|
||||
typedef unsigned char utf8proc_uint8_t;
|
||||
typedef short utf8proc_int16_t;
|
||||
typedef unsigned short utf8proc_uint16_t;
|
||||
typedef int utf8proc_int32_t;
|
||||
typedef unsigned int utf8proc_uint32_t;
|
||||
# ifdef _WIN64
|
||||
typedef __int64 utf8proc_ssize_t;
|
||||
typedef unsigned __int64 utf8proc_size_t;
|
||||
# else
|
||||
typedef int utf8proc_ssize_t;
|
||||
typedef unsigned int utf8proc_size_t;
|
||||
# endif
|
||||
# ifndef __cplusplus
|
||||
// emulate C99 bool
|
||||
typedef unsigned char utf8proc_bool;
|
||||
# ifndef __bool_true_false_are_defined
|
||||
# define false 0
|
||||
# define true 1
|
||||
# define __bool_true_false_are_defined 1
|
||||
# endif
|
||||
# else
|
||||
typedef bool utf8proc_bool;
|
||||
# endif
|
||||
#else
|
||||
# include <stddef.h>
|
||||
# include <stdbool.h>
|
||||
# include <inttypes.h>
|
||||
typedef int8_t utf8proc_int8_t;
|
||||
typedef uint8_t utf8proc_uint8_t;
|
||||
typedef int16_t utf8proc_int16_t;
|
||||
typedef uint16_t utf8proc_uint16_t;
|
||||
typedef int32_t utf8proc_int32_t;
|
||||
typedef uint32_t utf8proc_uint32_t;
|
||||
typedef size_t utf8proc_size_t;
|
||||
typedef ptrdiff_t utf8proc_ssize_t;
|
||||
typedef bool utf8proc_bool;
|
||||
#endif
|
||||
#include <limits.h>
|
||||
|
||||
#define UTF8PROC_DLLEXPORT
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#ifndef SSIZE_MAX
|
||||
#define SSIZE_MAX ((size_t)SIZE_MAX/2)
|
||||
#endif
|
||||
|
||||
#ifndef UINT16_MAX
|
||||
# define UINT16_MAX 65535U
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Option flags used by several functions in the library.
|
||||
*/
|
||||
typedef enum {
|
||||
/** The given UTF-8 input is NULL terminated. */
|
||||
UTF8PROC_NULLTERM = (1<<0),
|
||||
/** Unicode Versioning Stability has to be respected. */
|
||||
UTF8PROC_STABLE = (1<<1),
|
||||
/** Compatibility decomposition (i.e. formatting information is lost). */
|
||||
UTF8PROC_COMPAT = (1<<2),
|
||||
/** Return a result with decomposed characters. */
|
||||
UTF8PROC_COMPOSE = (1<<3),
|
||||
/** Return a result with decomposed characters. */
|
||||
UTF8PROC_DECOMPOSE = (1<<4),
|
||||
/** Strip "default ignorable characters" such as SOFT-HYPHEN or ZERO-WIDTH-SPACE. */
|
||||
UTF8PROC_IGNORE = (1<<5),
|
||||
/** Return an error, if the input contains unassigned codepoints. */
|
||||
UTF8PROC_REJECTNA = (1<<6),
|
||||
/**
|
||||
* Indicating that NLF-sequences (LF, CRLF, CR, NEL) are representing a
|
||||
* line break, and should be converted to the codepoint for line
|
||||
* separation (LS).
|
||||
*/
|
||||
UTF8PROC_NLF2LS = (1<<7),
|
||||
/**
|
||||
* Indicating that NLF-sequences are representing a paragraph break, and
|
||||
* should be converted to the codepoint for paragraph separation
|
||||
* (PS).
|
||||
*/
|
||||
UTF8PROC_NLF2PS = (1<<8),
|
||||
/** Indicating that the meaning of NLF-sequences is unknown. */
|
||||
UTF8PROC_NLF2LF = (UTF8PROC_NLF2LS | UTF8PROC_NLF2PS),
|
||||
/** Strips and/or convers control characters.
|
||||
*
|
||||
* NLF-sequences are transformed into space, except if one of the
|
||||
* NLF2LS/PS/LF options is given. HorizontalTab (HT) and FormFeed (FF)
|
||||
* are treated as a NLF-sequence in this case. All other control
|
||||
* characters are simply removed.
|
||||
*/
|
||||
UTF8PROC_STRIPCC = (1<<9),
|
||||
/**
|
||||
* Performs unicode case folding, to be able to do a case-insensitive
|
||||
* string comparison.
|
||||
*/
|
||||
UTF8PROC_CASEFOLD = (1<<10),
|
||||
/**
|
||||
* Inserts 0xFF bytes at the beginning of each sequence which is
|
||||
* representing a single grapheme cluster (see UAX#29).
|
||||
*/
|
||||
UTF8PROC_CHARBOUND = (1<<11),
|
||||
/** Lumps certain characters together.
|
||||
*
|
||||
* E.g. HYPHEN U+2010 and MINUS U+2212 to ASCII "-". See lump.md for details.
|
||||
*
|
||||
* If NLF2LF is set, this includes a transformation of paragraph and
|
||||
* line separators to ASCII line-feed (LF).
|
||||
*/
|
||||
UTF8PROC_LUMP = (1<<12),
|
||||
/** Strips all character markings.
|
||||
*
|
||||
* This includes non-spacing, spacing and enclosing (i.e. accents).
|
||||
* @note This option works only with @ref UTF8PROC_COMPOSE or
|
||||
* @ref UTF8PROC_DECOMPOSE
|
||||
*/
|
||||
UTF8PROC_STRIPMARK = (1<<13),
|
||||
} utf8proc_option_t;
|
||||
|
||||
/** @name Error codes
|
||||
* Error codes being returned by almost all functions.
|
||||
*/
|
||||
/** @{ */
|
||||
/** Memory could not be allocated. */
|
||||
#define UTF8PROC_ERROR_NOMEM -1
|
||||
/** The given string is too long to be processed. */
|
||||
#define UTF8PROC_ERROR_OVERFLOW -2
|
||||
/** The given string is not a legal UTF-8 string. */
|
||||
#define UTF8PROC_ERROR_INVALIDUTF8 -3
|
||||
/** The @ref UTF8PROC_REJECTNA flag was set and an unassigned codepoint was found. */
|
||||
#define UTF8PROC_ERROR_NOTASSIGNED -4
|
||||
/** Invalid options have been used. */
|
||||
#define UTF8PROC_ERROR_INVALIDOPTS -5
|
||||
/** @} */
|
||||
|
||||
/* @name Types */
|
||||
|
||||
/** Holds the value of a property. */
|
||||
typedef utf8proc_int16_t utf8proc_propval_t;
|
||||
|
||||
/** Struct containing information about a codepoint. */
|
||||
typedef struct utf8proc_property_struct {
|
||||
/**
|
||||
* Unicode category.
|
||||
* @see utf8proc_category_t.
|
||||
*/
|
||||
utf8proc_propval_t category;
|
||||
utf8proc_propval_t combining_class;
|
||||
/**
|
||||
* Bidirectional class.
|
||||
* @see utf8proc_bidi_class_t.
|
||||
*/
|
||||
utf8proc_propval_t bidi_class;
|
||||
/**
|
||||
* @anchor Decomposition type.
|
||||
* @see utf8proc_decomp_type_t.
|
||||
*/
|
||||
utf8proc_propval_t decomp_type;
|
||||
utf8proc_uint16_t decomp_seqindex;
|
||||
utf8proc_uint16_t casefold_seqindex;
|
||||
utf8proc_uint16_t uppercase_seqindex;
|
||||
utf8proc_uint16_t lowercase_seqindex;
|
||||
utf8proc_uint16_t titlecase_seqindex;
|
||||
utf8proc_uint16_t comb_index;
|
||||
unsigned bidi_mirrored:1;
|
||||
unsigned comp_exclusion:1;
|
||||
/**
|
||||
* Can this codepoint be ignored?
|
||||
*
|
||||
* Used by @ref utf8proc_decompose_char when @ref UTF8PROC_IGNORE is
|
||||
* passed as an option.
|
||||
*/
|
||||
unsigned ignorable:1;
|
||||
unsigned control_boundary:1;
|
||||
/** The width of the codepoint. */
|
||||
unsigned charwidth:2;
|
||||
unsigned pad:2;
|
||||
/**
|
||||
* Boundclass.
|
||||
* @see utf8proc_boundclass_t.
|
||||
*/
|
||||
unsigned boundclass:8;
|
||||
} utf8proc_property_t;
|
||||
|
||||
/** Unicode categories. */
|
||||
typedef enum {
|
||||
UTF8PROC_CATEGORY_CN = 0, /**< Other, not assigned */
|
||||
UTF8PROC_CATEGORY_LU = 1, /**< Letter, uppercase */
|
||||
UTF8PROC_CATEGORY_LL = 2, /**< Letter, lowercase */
|
||||
UTF8PROC_CATEGORY_LT = 3, /**< Letter, titlecase */
|
||||
UTF8PROC_CATEGORY_LM = 4, /**< Letter, modifier */
|
||||
UTF8PROC_CATEGORY_LO = 5, /**< Letter, other */
|
||||
UTF8PROC_CATEGORY_MN = 6, /**< Mark, nonspacing */
|
||||
UTF8PROC_CATEGORY_MC = 7, /**< Mark, spacing combining */
|
||||
UTF8PROC_CATEGORY_ME = 8, /**< Mark, enclosing */
|
||||
UTF8PROC_CATEGORY_ND = 9, /**< Number, decimal digit */
|
||||
UTF8PROC_CATEGORY_NL = 10, /**< Number, letter */
|
||||
UTF8PROC_CATEGORY_NO = 11, /**< Number, other */
|
||||
UTF8PROC_CATEGORY_PC = 12, /**< Punctuation, connector */
|
||||
UTF8PROC_CATEGORY_PD = 13, /**< Punctuation, dash */
|
||||
UTF8PROC_CATEGORY_PS = 14, /**< Punctuation, open */
|
||||
UTF8PROC_CATEGORY_PE = 15, /**< Punctuation, close */
|
||||
UTF8PROC_CATEGORY_PI = 16, /**< Punctuation, initial quote */
|
||||
UTF8PROC_CATEGORY_PF = 17, /**< Punctuation, final quote */
|
||||
UTF8PROC_CATEGORY_PO = 18, /**< Punctuation, other */
|
||||
UTF8PROC_CATEGORY_SM = 19, /**< Symbol, math */
|
||||
UTF8PROC_CATEGORY_SC = 20, /**< Symbol, currency */
|
||||
UTF8PROC_CATEGORY_SK = 21, /**< Symbol, modifier */
|
||||
UTF8PROC_CATEGORY_SO = 22, /**< Symbol, other */
|
||||
UTF8PROC_CATEGORY_ZS = 23, /**< Separator, space */
|
||||
UTF8PROC_CATEGORY_ZL = 24, /**< Separator, line */
|
||||
UTF8PROC_CATEGORY_ZP = 25, /**< Separator, paragraph */
|
||||
UTF8PROC_CATEGORY_CC = 26, /**< Other, control */
|
||||
UTF8PROC_CATEGORY_CF = 27, /**< Other, format */
|
||||
UTF8PROC_CATEGORY_CS = 28, /**< Other, surrogate */
|
||||
UTF8PROC_CATEGORY_CO = 29, /**< Other, private use */
|
||||
} utf8proc_category_t;
|
||||
|
||||
/** Bidirectional character classes. */
|
||||
typedef enum {
|
||||
UTF8PROC_BIDI_CLASS_L = 1, /**< Left-to-Right */
|
||||
UTF8PROC_BIDI_CLASS_LRE = 2, /**< Left-to-Right Embedding */
|
||||
UTF8PROC_BIDI_CLASS_LRO = 3, /**< Left-to-Right Override */
|
||||
UTF8PROC_BIDI_CLASS_R = 4, /**< Right-to-Left */
|
||||
UTF8PROC_BIDI_CLASS_AL = 5, /**< Right-to-Left Arabic */
|
||||
UTF8PROC_BIDI_CLASS_RLE = 6, /**< Right-to-Left Embedding */
|
||||
UTF8PROC_BIDI_CLASS_RLO = 7, /**< Right-to-Left Override */
|
||||
UTF8PROC_BIDI_CLASS_PDF = 8, /**< Pop Directional Format */
|
||||
UTF8PROC_BIDI_CLASS_EN = 9, /**< European Number */
|
||||
UTF8PROC_BIDI_CLASS_ES = 10, /**< European Separator */
|
||||
UTF8PROC_BIDI_CLASS_ET = 11, /**< European Number Terminator */
|
||||
UTF8PROC_BIDI_CLASS_AN = 12, /**< Arabic Number */
|
||||
UTF8PROC_BIDI_CLASS_CS = 13, /**< Common Number Separator */
|
||||
UTF8PROC_BIDI_CLASS_NSM = 14, /**< Nonspacing Mark */
|
||||
UTF8PROC_BIDI_CLASS_BN = 15, /**< Boundary Neutral */
|
||||
UTF8PROC_BIDI_CLASS_B = 16, /**< Paragraph Separator */
|
||||
UTF8PROC_BIDI_CLASS_S = 17, /**< Segment Separator */
|
||||
UTF8PROC_BIDI_CLASS_WS = 18, /**< Whitespace */
|
||||
UTF8PROC_BIDI_CLASS_ON = 19, /**< Other Neutrals */
|
||||
UTF8PROC_BIDI_CLASS_LRI = 20, /**< Left-to-Right Isolate */
|
||||
UTF8PROC_BIDI_CLASS_RLI = 21, /**< Right-to-Left Isolate */
|
||||
UTF8PROC_BIDI_CLASS_FSI = 22, /**< First Strong Isolate */
|
||||
UTF8PROC_BIDI_CLASS_PDI = 23, /**< Pop Directional Isolate */
|
||||
} utf8proc_bidi_class_t;
|
||||
|
||||
/** Decomposition type. */
|
||||
typedef enum {
|
||||
UTF8PROC_DECOMP_TYPE_FONT = 1, /**< Font */
|
||||
UTF8PROC_DECOMP_TYPE_NOBREAK = 2, /**< Nobreak */
|
||||
UTF8PROC_DECOMP_TYPE_INITIAL = 3, /**< Initial */
|
||||
UTF8PROC_DECOMP_TYPE_MEDIAL = 4, /**< Medial */
|
||||
UTF8PROC_DECOMP_TYPE_FINAL = 5, /**< Final */
|
||||
UTF8PROC_DECOMP_TYPE_ISOLATED = 6, /**< Isolated */
|
||||
UTF8PROC_DECOMP_TYPE_CIRCLE = 7, /**< Circle */
|
||||
UTF8PROC_DECOMP_TYPE_SUPER = 8, /**< Super */
|
||||
UTF8PROC_DECOMP_TYPE_SUB = 9, /**< Sub */
|
||||
UTF8PROC_DECOMP_TYPE_VERTICAL = 10, /**< Vertical */
|
||||
UTF8PROC_DECOMP_TYPE_WIDE = 11, /**< Wide */
|
||||
UTF8PROC_DECOMP_TYPE_NARROW = 12, /**< Narrow */
|
||||
UTF8PROC_DECOMP_TYPE_SMALL = 13, /**< Small */
|
||||
UTF8PROC_DECOMP_TYPE_SQUARE = 14, /**< Square */
|
||||
UTF8PROC_DECOMP_TYPE_FRACTION = 15, /**< Fraction */
|
||||
UTF8PROC_DECOMP_TYPE_COMPAT = 16, /**< Compat */
|
||||
} utf8proc_decomp_type_t;
|
||||
|
||||
/** Boundclass property. (TR29) */
|
||||
typedef enum {
|
||||
UTF8PROC_BOUNDCLASS_START = 0, /**< Start */
|
||||
UTF8PROC_BOUNDCLASS_OTHER = 1, /**< Other */
|
||||
UTF8PROC_BOUNDCLASS_CR = 2, /**< Cr */
|
||||
UTF8PROC_BOUNDCLASS_LF = 3, /**< Lf */
|
||||
UTF8PROC_BOUNDCLASS_CONTROL = 4, /**< Control */
|
||||
UTF8PROC_BOUNDCLASS_EXTEND = 5, /**< Extend */
|
||||
UTF8PROC_BOUNDCLASS_L = 6, /**< L */
|
||||
UTF8PROC_BOUNDCLASS_V = 7, /**< V */
|
||||
UTF8PROC_BOUNDCLASS_T = 8, /**< T */
|
||||
UTF8PROC_BOUNDCLASS_LV = 9, /**< Lv */
|
||||
UTF8PROC_BOUNDCLASS_LVT = 10, /**< Lvt */
|
||||
UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR = 11, /**< Regional indicator */
|
||||
UTF8PROC_BOUNDCLASS_SPACINGMARK = 12, /**< Spacingmark */
|
||||
UTF8PROC_BOUNDCLASS_PREPEND = 13, /**< Prepend */
|
||||
UTF8PROC_BOUNDCLASS_ZWJ = 14, /**< Zero Width Joiner */
|
||||
UTF8PROC_BOUNDCLASS_E_BASE = 15, /**< Emoji Base */
|
||||
UTF8PROC_BOUNDCLASS_E_MODIFIER = 16, /**< Emoji Modifier */
|
||||
UTF8PROC_BOUNDCLASS_GLUE_AFTER_ZWJ = 17, /**< Glue_After_ZWJ */
|
||||
UTF8PROC_BOUNDCLASS_E_BASE_GAZ = 18, /**< E_BASE + GLUE_AFTER_ZJW */
|
||||
} utf8proc_boundclass_t;
|
||||
|
||||
/**
|
||||
* Function pointer type passed to @ref utf8proc_map_custom and
|
||||
* @ref utf8proc_decompose_custom, which is used to specify a user-defined
|
||||
* mapping of codepoints to be applied in conjunction with other mappings.
|
||||
*/
|
||||
typedef utf8proc_int32_t (*utf8proc_custom_func)(utf8proc_int32_t codepoint, void *data);
|
||||
|
||||
/**
|
||||
* Array containing the byte lengths of a UTF-8 encoded codepoint based
|
||||
* on the first byte.
|
||||
*/
|
||||
UTF8PROC_DLLEXPORT extern const utf8proc_int8_t utf8proc_utf8class[256];
|
||||
|
||||
/**
|
||||
* Returns the utf8proc API version as a string MAJOR.MINOR.PATCH
|
||||
* (http://semver.org format), possibly with a "-dev" suffix for
|
||||
* development versions.
|
||||
*/
|
||||
UTF8PROC_DLLEXPORT const char *utf8proc_version(void);
|
||||
|
||||
/**
|
||||
* Returns an informative error string for the given utf8proc error code
|
||||
* (e.g. the error codes returned by @ref utf8proc_map).
|
||||
*/
|
||||
UTF8PROC_DLLEXPORT const char *utf8proc_errmsg(utf8proc_ssize_t errcode);
|
||||
|
||||
/**
|
||||
* Reads a single codepoint from the UTF-8 sequence being pointed to by `str`.
|
||||
* The maximum number of bytes read is `strlen`, unless `strlen` is
|
||||
* negative (in which case up to 4 bytes are read).
|
||||
*
|
||||
* If a valid codepoint could be read, it is stored in the variable
|
||||
* pointed to by `codepoint_ref`, otherwise that variable will be set to -1.
|
||||
* In case of success, the number of bytes read is returned; otherwise, a
|
||||
* negative error code is returned.
|
||||
*/
|
||||
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_iterate(const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *codepoint_ref);
|
||||
|
||||
/**
|
||||
* Check if a codepoint is valid (regardless of whether it has been
|
||||
* assigned a value by the current Unicode standard).
|
||||
*
|
||||
* @return 1 if the given `codepoint` is valid and otherwise return 0.
|
||||
*/
|
||||
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_codepoint_valid(utf8proc_int32_t codepoint);
|
||||
|
||||
/**
|
||||
* Encodes the codepoint as an UTF-8 string in the byte array pointed
|
||||
* to by `dst`. This array must be at least 4 bytes long.
|
||||
*
|
||||
* In case of success the number of bytes written is returned, and
|
||||
* otherwise 0 is returned.
|
||||
*
|
||||
* This function does not check whether `codepoint` is valid Unicode.
|
||||
*/
|
||||
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t codepoint, utf8proc_uint8_t *dst);
|
||||
|
||||
/**
|
||||
* Look up the properties for a given codepoint.
|
||||
*
|
||||
* @param codepoint The Unicode codepoint.
|
||||
*
|
||||
* @returns
|
||||
* A pointer to a (constant) struct containing information about
|
||||
* the codepoint.
|
||||
* @par
|
||||
* If the codepoint is unassigned or invalid, a pointer to a special struct is
|
||||
* returned in which `category` is 0 (@ref UTF8PROC_CATEGORY_CN).
|
||||
*/
|
||||
UTF8PROC_DLLEXPORT const utf8proc_property_t *utf8proc_get_property(utf8proc_int32_t codepoint);
|
||||
|
||||
/** Decompose a codepoint into an array of codepoints.
|
||||
*
|
||||
* @param codepoint the codepoint.
|
||||
* @param dst the destination buffer.
|
||||
* @param bufsize the size of the destination buffer.
|
||||
* @param options one or more of the following flags:
|
||||
* - @ref UTF8PROC_REJECTNA - return an error `codepoint` is unassigned
|
||||
* - @ref UTF8PROC_IGNORE - strip "default ignorable" codepoints
|
||||
* - @ref UTF8PROC_CASEFOLD - apply Unicode casefolding
|
||||
* - @ref UTF8PROC_COMPAT - replace certain codepoints with their
|
||||
* compatibility decomposition
|
||||
* - @ref UTF8PROC_CHARBOUND - insert 0xFF bytes before each grapheme cluster
|
||||
* - @ref UTF8PROC_LUMP - lump certain different codepoints together
|
||||
* - @ref UTF8PROC_STRIPMARK - remove all character marks
|
||||
* @param last_boundclass
|
||||
* Pointer to an integer variable containing
|
||||
* the previous codepoint's boundary class if the @ref UTF8PROC_CHARBOUND
|
||||
* option is used. Otherwise, this parameter is ignored.
|
||||
*
|
||||
* @return
|
||||
* In case of success, the number of codepoints written is returned; in case
|
||||
* of an error, a negative error code is returned (@ref utf8proc_errmsg).
|
||||
* @par
|
||||
* If the number of written codepoints would be bigger than `bufsize`, the
|
||||
* required buffer size is returned, while the buffer will be overwritten with
|
||||
* undefined data.
|
||||
*/
|
||||
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char(
|
||||
utf8proc_int32_t codepoint, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize,
|
||||
utf8proc_option_t options, int *last_boundclass
|
||||
);
|
||||
|
||||
/**
|
||||
* The same as @ref utf8proc_decompose_char, but acts on a whole UTF-8
|
||||
* string and orders the decomposed sequences correctly.
|
||||
*
|
||||
* If the @ref UTF8PROC_NULLTERM flag in `options` is set, processing
|
||||
* will be stopped, when a NULL byte is encounted, otherwise `strlen`
|
||||
* bytes are processed. The result (in the form of 32-bit unicode
|
||||
* codepoints) is written into the buffer being pointed to by
|
||||
* `buffer` (which must contain at least `bufsize` entries). In case of
|
||||
* success, the number of codepoints written is returned; in case of an
|
||||
* error, a negative error code is returned (@ref utf8proc_errmsg).
|
||||
* See @ref utf8proc_decompose_custom to supply additional transformations.
|
||||
*
|
||||
* If the number of written codepoints would be bigger than `bufsize`, the
|
||||
* required buffer size is returned, while the buffer will be overwritten with
|
||||
* undefined data.
|
||||
*/
|
||||
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose(
|
||||
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
|
||||
utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options
|
||||
);
|
||||
|
||||
/**
|
||||
* The same as @ref utf8proc_decompose, but also takes a `custom_func` mapping function
|
||||
* that is called on each codepoint in `str` before any other transformations
|
||||
* (along with a `custom_data` pointer that is passed through to `custom_func`).
|
||||
* The `custom_func` argument is ignored if it is `NULL`. See also @ref utf8proc_map_custom.
|
||||
*/
|
||||
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_custom(
|
||||
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
|
||||
utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options,
|
||||
utf8proc_custom_func custom_func, void *custom_data
|
||||
);
|
||||
|
||||
/**
|
||||
* Normalizes the sequence of `length` codepoints pointed to by `buffer`
|
||||
* in-place (i.e., the result is also stored in `buffer`).
|
||||
*
|
||||
* @param buffer the (native-endian UTF-32) unicode codepoints to re-encode.
|
||||
* @param length the length (in codepoints) of the buffer.
|
||||
* @param options a bitwise or (`|`) of one or more of the following flags:
|
||||
* - @ref UTF8PROC_NLF2LS - convert LF, CRLF, CR and NEL into LS
|
||||
* - @ref UTF8PROC_NLF2PS - convert LF, CRLF, CR and NEL into PS
|
||||
* - @ref UTF8PROC_NLF2LF - convert LF, CRLF, CR and NEL into LF
|
||||
* - @ref UTF8PROC_STRIPCC - strip or convert all non-affected control characters
|
||||
* - @ref UTF8PROC_COMPOSE - try to combine decomposed codepoints into composite
|
||||
* codepoints
|
||||
* - @ref UTF8PROC_STABLE - prohibit combining characters that would violate
|
||||
* the unicode versioning stability
|
||||
*
|
||||
* @return
|
||||
* In case of success, the length (in codepoints) of the normalized UTF-32 string is
|
||||
* returned; otherwise, a negative error code is returned (@ref utf8proc_errmsg).
|
||||
*
|
||||
* @warning The entries of the array pointed to by `str` have to be in the
|
||||
* range `0x0000` to `0x10FFFF`. Otherwise, the program might crash!
|
||||
*/
|
||||
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_normalize_utf32(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options);
|
||||
|
||||
/**
|
||||
* Reencodes the sequence of `length` codepoints pointed to by `buffer`
|
||||
* UTF-8 data in-place (i.e., the result is also stored in `buffer`).
|
||||
* Can optionally normalize the UTF-32 sequence prior to UTF-8 conversion.
|
||||
*
|
||||
* @param buffer the (native-endian UTF-32) unicode codepoints to re-encode.
|
||||
* @param length the length (in codepoints) of the buffer.
|
||||
* @param options a bitwise or (`|`) of one or more of the following flags:
|
||||
* - @ref UTF8PROC_NLF2LS - convert LF, CRLF, CR and NEL into LS
|
||||
* - @ref UTF8PROC_NLF2PS - convert LF, CRLF, CR and NEL into PS
|
||||
* - @ref UTF8PROC_NLF2LF - convert LF, CRLF, CR and NEL into LF
|
||||
* - @ref UTF8PROC_STRIPCC - strip or convert all non-affected control characters
|
||||
* - @ref UTF8PROC_COMPOSE - try to combine decomposed codepoints into composite
|
||||
* codepoints
|
||||
* - @ref UTF8PROC_STABLE - prohibit combining characters that would violate
|
||||
* the unicode versioning stability
|
||||
* - @ref UTF8PROC_CHARBOUND - insert 0xFF bytes before each grapheme cluster
|
||||
*
|
||||
* @return
|
||||
* In case of success, the length (in bytes) of the resulting nul-terminated
|
||||
* UTF-8 string is returned; otherwise, a negative error code is returned
|
||||
* (@ref utf8proc_errmsg).
|
||||
*
|
||||
* @warning The amount of free space pointed to by `buffer` must
|
||||
* exceed the amount of the input data by one byte, and the
|
||||
* entries of the array pointed to by `str` have to be in the
|
||||
* range `0x0000` to `0x10FFFF`. Otherwise, the program might crash!
|
||||
*/
|
||||
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options);
|
||||
|
||||
/**
|
||||
* Given a pair of consecutive codepoints, return whether a grapheme break is
|
||||
* permitted between them (as defined by the extended grapheme clusters in UAX#29).
|
||||
*
|
||||
* @param state Beginning with Version 29 (Unicode 9.0.0), this algorithm requires
|
||||
* state to break graphemes. This state can be passed in as a pointer
|
||||
* in the `state` argument and should initially be set to 0. If the
|
||||
* state is not passed in (i.e. a null pointer is passed), UAX#29 rules
|
||||
* GB10/12/13 which require this state will not be applied, essentially
|
||||
* matching the rules in Unicode 8.0.0.
|
||||
*
|
||||
* @warning If the state parameter is used, `utf8proc_grapheme_break_stateful` must
|
||||
* be called IN ORDER on ALL potential breaks in a string.
|
||||
*/
|
||||
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break_stateful(
|
||||
utf8proc_int32_t codepoint1, utf8proc_int32_t codepoint2, utf8proc_int32_t *state);
|
||||
|
||||
/**
|
||||
* Same as @ref utf8proc_grapheme_break_stateful, except without support for the
|
||||
* Unicode 9 additions to the algorithm. Supported for legacy reasons.
|
||||
*/
|
||||
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break(
|
||||
utf8proc_int32_t codepoint1, utf8proc_int32_t codepoint2);
|
||||
|
||||
|
||||
/**
|
||||
* Given a codepoint `c`, return the codepoint of the corresponding
|
||||
* lower-case character, if any; otherwise (if there is no lower-case
|
||||
* variant, or if `c` is not a valid codepoint) return `c`.
|
||||
*/
|
||||
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_tolower(utf8proc_int32_t c);
|
||||
|
||||
/**
|
||||
* Given a codepoint `c`, return the codepoint of the corresponding
|
||||
* upper-case character, if any; otherwise (if there is no upper-case
|
||||
* variant, or if `c` is not a valid codepoint) return `c`.
|
||||
*/
|
||||
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_toupper(utf8proc_int32_t c);
|
||||
|
||||
/**
|
||||
* Given a codepoint `c`, return the codepoint of the corresponding
|
||||
* title-case character, if any; otherwise (if there is no title-case
|
||||
* variant, or if `c` is not a valid codepoint) return `c`.
|
||||
*/
|
||||
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_totitle(utf8proc_int32_t c);
|
||||
|
||||
/**
|
||||
* Given a codepoint, return a character width analogous to `wcwidth(codepoint)`,
|
||||
* except that a width of 0 is returned for non-printable codepoints
|
||||
* instead of -1 as in `wcwidth`.
|
||||
*
|
||||
* @note
|
||||
* If you want to check for particular types of non-printable characters,
|
||||
* (analogous to `isprint` or `iscntrl`), use @ref utf8proc_category. */
|
||||
UTF8PROC_DLLEXPORT int utf8proc_charwidth(utf8proc_int32_t codepoint);
|
||||
|
||||
/**
|
||||
* Return the Unicode category for the codepoint (one of the
|
||||
* @ref utf8proc_category_t constants.)
|
||||
*/
|
||||
UTF8PROC_DLLEXPORT utf8proc_category_t utf8proc_category(utf8proc_int32_t codepoint);
|
||||
|
||||
/**
|
||||
* Return the two-letter (nul-terminated) Unicode category string for
|
||||
* the codepoint (e.g. `"Lu"` or `"Co"`).
|
||||
*/
|
||||
UTF8PROC_DLLEXPORT const char *utf8proc_category_string(utf8proc_int32_t codepoint);
|
||||
|
||||
/**
|
||||
* Maps the given UTF-8 string pointed to by `str` to a new UTF-8
|
||||
* string, allocated dynamically by `malloc` and returned via `dstptr`.
|
||||
*
|
||||
* If the @ref UTF8PROC_NULLTERM flag in the `options` field is set,
|
||||
* the length is determined by a NULL terminator, otherwise the
|
||||
* parameter `strlen` is evaluated to determine the string length, but
|
||||
* in any case the result will be NULL terminated (though it might
|
||||
* contain NULL characters with the string if `str` contained NULL
|
||||
* characters). Other flags in the `options` field are passed to the
|
||||
* functions defined above, and regarded as described. See also
|
||||
* @ref utfproc_map_custom to supply a custom codepoint transformation.
|
||||
*
|
||||
* In case of success the length of the new string is returned,
|
||||
* otherwise a negative error code is returned.
|
||||
*
|
||||
* @note The memory of the new UTF-8 string will have been allocated
|
||||
* with `malloc`, and should therefore be deallocated with `free`.
|
||||
*/
|
||||
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map(
|
||||
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options
|
||||
);
|
||||
|
||||
/**
|
||||
* Like @ref utf8proc_map, but also takes a `custom_func` mapping function
|
||||
* that is called on each codepoint in `str` before any other transformations
|
||||
* (along with a `custom_data` pointer that is passed through to `custom_func`).
|
||||
* The `custom_func` argument is ignored if it is `NULL`.
|
||||
*/
|
||||
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map_custom(
|
||||
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options,
|
||||
utf8proc_custom_func custom_func, void *custom_data
|
||||
);
|
||||
|
||||
/** @name Unicode normalization
|
||||
*
|
||||
* Returns a pointer to newly allocated memory of a NFD, NFC, NFKD or NFKC
|
||||
* normalized version of the null-terminated string `str`. These
|
||||
* are shortcuts to calling @ref utf8proc_map with @ref UTF8PROC_NULLTERM
|
||||
* combined with @ref UTF8PROC_STABLE and flags indicating the normalization.
|
||||
*/
|
||||
/** @{ */
|
||||
/** NFD normalization (@ref UTF8PROC_DECOMPOSE). */
|
||||
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFD(const utf8proc_uint8_t *str);
|
||||
/** NFC normalization (@ref UTF8PROC_COMPOSE). */
|
||||
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFC(const utf8proc_uint8_t *str);
|
||||
/** NFKD normalization (@ref UTF8PROC_DECOMPOSE and @ref UTF8PROC_COMPAT). */
|
||||
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKD(const utf8proc_uint8_t *str);
|
||||
/** NFKC normalization (@ref UTF8PROC_COMPOSE and @ref UTF8PROC_COMPAT). */
|
||||
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC(const utf8proc_uint8_t *str);
|
||||
/** @} */
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,20 @@
|
|||
|
||||
function (disallow_intree_builds)
|
||||
# Adapted from LLVM's toplevel CMakeLists.txt file
|
||||
if( CMAKE_SOURCE_DIR STREQUAL CMAKE_BINARY_DIR AND NOT MSVC_IDE )
|
||||
message(FATAL_ERROR "
|
||||
In-source builds are not allowed. CMake would overwrite the
|
||||
makefiles distributed with utf8proc. Please create a directory
|
||||
and run cmake from there. Building in a subdirectory is
|
||||
fine, e.g.:
|
||||
|
||||
mkdir build
|
||||
cd build
|
||||
cmake ..
|
||||
|
||||
This process created the file `CMakeCache.txt' and the
|
||||
directory `CMakeFiles'. Please delete them.
|
||||
|
||||
")
|
||||
endif()
|
||||
endfunction()
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
diff --git a/lib/utf8proc-2a2f97e1/utf8proc.h b/lib/utf8proc-2a2f97e1/utf8proc.h
|
||||
index 64155a1..2fca528 100644
|
||||
--- a/lib/utf8proc-2a2f97e1/utf8proc.h
|
||||
+++ b/lib/utf8proc-2a2f97e1/utf8proc.h
|
||||
@@ -120,17 +120,7 @@ typedef bool utf8proc_bool;
|
||||
#endif
|
||||
#include <limits.h>
|
||||
|
||||
-#ifdef _WIN32
|
||||
-# ifdef UTF8PROC_EXPORTS
|
||||
-# define UTF8PROC_DLLEXPORT __declspec(dllexport)
|
||||
-# else
|
||||
-# define UTF8PROC_DLLEXPORT __declspec(dllimport)
|
||||
-# endif
|
||||
-#elif __GNUC__ >= 4
|
||||
-# define UTF8PROC_DLLEXPORT __attribute__ ((visibility("default")))
|
||||
-#else
|
||||
-# define UTF8PROC_DLLEXPORT
|
||||
-#endif
|
||||
+#define UTF8PROC_DLLEXPORT
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
cmake_minimum_required (VERSION 2.8.0)
|
||||
project (utf8cpp)
|
||||
include_directories ("${PROJECT_SOURCE_DIR}/source")
|
||||
|
||||
add_executable(smoke ${PROJECT_SOURCE_DIR}/test_drivers/smoke_test/test.cpp)
|
||||
add_executable(negative ${PROJECT_SOURCE_DIR}/test_drivers/negative/negative.cpp)
|
||||
add_executable(utf8reader ${PROJECT_SOURCE_DIR}/test_drivers/utf8reader/utf8reader.cpp)
|
||||
|
||||
add_executable(docsample ${PROJECT_SOURCE_DIR}/samples/docsample.cpp)
|
||||
|
||||
enable_testing()
|
||||
add_test(smoke_test smoke)
|
||||
add_test(negative_test negative ${PROJECT_SOURCE_DIR}/test_data/negative/utf8_invalid.txt)
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,52 @@
|
|||
#include "../source/utf8.h"
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
|
||||
using namespace std;
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
if (argc != 2) {
|
||||
cout << "\nUsage: docsample filename\n";
|
||||
return 0;
|
||||
}
|
||||
const char* test_file_path = argv[1];
|
||||
// Open the test file (must be UTF-8 encoded)
|
||||
ifstream fs8(test_file_path);
|
||||
if (!fs8.is_open()) {
|
||||
cout << "Could not open " << test_file_path << endl;
|
||||
return 0;
|
||||
}
|
||||
|
||||
unsigned line_count = 1;
|
||||
string line;
|
||||
// Play with all the lines in the file
|
||||
while (getline(fs8, line)) {
|
||||
// check for invalid utf-8 (for a simple yes/no check, there is also utf8::is_valid function)
|
||||
string::iterator end_it = utf8::find_invalid(line.begin(), line.end());
|
||||
if (end_it != line.end()) {
|
||||
cout << "Invalid UTF-8 encoding detected at line " << line_count << "\n";
|
||||
cout << "This part is fine: " << string(line.begin(), end_it) << "\n";
|
||||
}
|
||||
// Get the line length (at least for the valid part)
|
||||
int length = utf8::distance(line.begin(), end_it);
|
||||
cout << "Length of line " << line_count << " is " << length << "\n";
|
||||
|
||||
// Convert it to utf-16
|
||||
vector<unsigned short> utf16line;
|
||||
utf8::utf8to16(line.begin(), end_it, back_inserter(utf16line));
|
||||
// And back to utf-8;
|
||||
string utf8line;
|
||||
utf8::utf16to8(utf16line.begin(), utf16line.end(), back_inserter(utf8line));
|
||||
// Confirm that the conversion went OK:
|
||||
if (utf8line != string(line.begin(), end_it))
|
||||
cout << "Error in UTF-16 conversion at line: " << line_count << "\n";
|
||||
|
||||
line_count++;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
// Copyright 2006 Nemanja Trifunovic
|
||||
|
||||
/*
|
||||
Permission is hereby granted, free of charge, to any person or organization
|
||||
obtaining a copy of the software and accompanying documentation covered by
|
||||
this license (the "Software") to use, reproduce, display, distribute,
|
||||
execute, and transmit the Software, and to prepare derivative works of the
|
||||
Software, and to permit third-parties to whom the Software is furnished to
|
||||
do so, all subject to the following:
|
||||
|
||||
The copyright notices in the Software and this entire statement, including
|
||||
the above license grant, this restriction and the following disclaimer,
|
||||
must be included in all copies of the Software, in whole or in part, and
|
||||
all derivative works of the Software, unless such copies or derivative
|
||||
works are solely in the form of machine-executable object code generated by
|
||||
a source language processor.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
|
||||
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
|
||||
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
|
||||
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
|
||||
#ifndef UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||
#define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||
|
||||
#include "utf8/checked.h"
|
||||
#include "utf8/unchecked.h"
|
||||
|
||||
#endif // header guard
|
||||
|
|
@ -0,0 +1,327 @@
|
|||
// Copyright 2006-2016 Nemanja Trifunovic
|
||||
|
||||
/*
|
||||
Permission is hereby granted, free of charge, to any person or organization
|
||||
obtaining a copy of the software and accompanying documentation covered by
|
||||
this license (the "Software") to use, reproduce, display, distribute,
|
||||
execute, and transmit the Software, and to prepare derivative works of the
|
||||
Software, and to permit third-parties to whom the Software is furnished to
|
||||
do so, all subject to the following:
|
||||
|
||||
The copyright notices in the Software and this entire statement, including
|
||||
the above license grant, this restriction and the following disclaimer,
|
||||
must be included in all copies of the Software, in whole or in part, and
|
||||
all derivative works of the Software, unless such copies or derivative
|
||||
works are solely in the form of machine-executable object code generated by
|
||||
a source language processor.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
|
||||
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
|
||||
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
|
||||
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
|
||||
#ifndef UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||
#define UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||
|
||||
#include "core.h"
|
||||
#include <stdexcept>
|
||||
|
||||
namespace utf8
|
||||
{
|
||||
// Base for the exceptions that may be thrown from the library
|
||||
class exception : public ::std::exception {
|
||||
};
|
||||
|
||||
// Exceptions that may be thrown from the library functions.
|
||||
class invalid_code_point : public exception {
|
||||
uint32_t cp;
|
||||
public:
|
||||
invalid_code_point(uint32_t codepoint) : cp(codepoint) {}
|
||||
virtual const char* what() const throw() { return "Invalid code point"; }
|
||||
uint32_t code_point() const {return cp;}
|
||||
};
|
||||
|
||||
class invalid_utf8 : public exception {
|
||||
uint8_t u8;
|
||||
public:
|
||||
invalid_utf8 (uint8_t u) : u8(u) {}
|
||||
virtual const char* what() const throw() { return "Invalid UTF-8"; }
|
||||
uint8_t utf8_octet() const {return u8;}
|
||||
};
|
||||
|
||||
class invalid_utf16 : public exception {
|
||||
uint16_t u16;
|
||||
public:
|
||||
invalid_utf16 (uint16_t u) : u16(u) {}
|
||||
virtual const char* what() const throw() { return "Invalid UTF-16"; }
|
||||
uint16_t utf16_word() const {return u16;}
|
||||
};
|
||||
|
||||
class not_enough_room : public exception {
|
||||
public:
|
||||
virtual const char* what() const throw() { return "Not enough space"; }
|
||||
};
|
||||
|
||||
/// The library API - functions intended to be called by the users
|
||||
|
||||
template <typename octet_iterator>
|
||||
octet_iterator append(uint32_t cp, octet_iterator result)
|
||||
{
|
||||
if (!utf8::internal::is_code_point_valid(cp))
|
||||
throw invalid_code_point(cp);
|
||||
|
||||
if (cp < 0x80) // one octet
|
||||
*(result++) = static_cast<uint8_t>(cp);
|
||||
else if (cp < 0x800) { // two octets
|
||||
*(result++) = static_cast<uint8_t>((cp >> 6) | 0xc0);
|
||||
*(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
|
||||
}
|
||||
else if (cp < 0x10000) { // three octets
|
||||
*(result++) = static_cast<uint8_t>((cp >> 12) | 0xe0);
|
||||
*(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80);
|
||||
*(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
|
||||
}
|
||||
else { // four octets
|
||||
*(result++) = static_cast<uint8_t>((cp >> 18) | 0xf0);
|
||||
*(result++) = static_cast<uint8_t>(((cp >> 12) & 0x3f) | 0x80);
|
||||
*(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80);
|
||||
*(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
template <typename octet_iterator, typename output_iterator>
|
||||
output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement)
|
||||
{
|
||||
while (start != end) {
|
||||
octet_iterator sequence_start = start;
|
||||
internal::utf_error err_code = utf8::internal::validate_next(start, end);
|
||||
switch (err_code) {
|
||||
case internal::UTF8_OK :
|
||||
for (octet_iterator it = sequence_start; it != start; ++it)
|
||||
*out++ = *it;
|
||||
break;
|
||||
case internal::NOT_ENOUGH_ROOM:
|
||||
throw not_enough_room();
|
||||
case internal::INVALID_LEAD:
|
||||
out = utf8::append (replacement, out);
|
||||
++start;
|
||||
break;
|
||||
case internal::INCOMPLETE_SEQUENCE:
|
||||
case internal::OVERLONG_SEQUENCE:
|
||||
case internal::INVALID_CODE_POINT:
|
||||
out = utf8::append (replacement, out);
|
||||
++start;
|
||||
// just one replacement mark for the sequence
|
||||
while (start != end && utf8::internal::is_trail(*start))
|
||||
++start;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
template <typename octet_iterator, typename output_iterator>
|
||||
inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out)
|
||||
{
|
||||
static const uint32_t replacement_marker = utf8::internal::mask16(0xfffd);
|
||||
return utf8::replace_invalid(start, end, out, replacement_marker);
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
uint32_t next(octet_iterator& it, octet_iterator end)
|
||||
{
|
||||
uint32_t cp = 0;
|
||||
internal::utf_error err_code = utf8::internal::validate_next(it, end, cp);
|
||||
switch (err_code) {
|
||||
case internal::UTF8_OK :
|
||||
break;
|
||||
case internal::NOT_ENOUGH_ROOM :
|
||||
throw not_enough_room();
|
||||
case internal::INVALID_LEAD :
|
||||
case internal::INCOMPLETE_SEQUENCE :
|
||||
case internal::OVERLONG_SEQUENCE :
|
||||
throw invalid_utf8(*it);
|
||||
case internal::INVALID_CODE_POINT :
|
||||
throw invalid_code_point(cp);
|
||||
}
|
||||
return cp;
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
uint32_t peek_next(octet_iterator it, octet_iterator end)
|
||||
{
|
||||
return utf8::next(it, end);
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
uint32_t prior(octet_iterator& it, octet_iterator start)
|
||||
{
|
||||
// can't do much if it == start
|
||||
if (it == start)
|
||||
throw not_enough_room();
|
||||
|
||||
octet_iterator end = it;
|
||||
// Go back until we hit either a lead octet or start
|
||||
while (utf8::internal::is_trail(*(--it)))
|
||||
if (it == start)
|
||||
throw invalid_utf8(*it); // error - no lead byte in the sequence
|
||||
return utf8::peek_next(it, end);
|
||||
}
|
||||
|
||||
/// Deprecated in versions that include "prior"
|
||||
template <typename octet_iterator>
|
||||
uint32_t previous(octet_iterator& it, octet_iterator pass_start)
|
||||
{
|
||||
octet_iterator end = it;
|
||||
while (utf8::internal::is_trail(*(--it)))
|
||||
if (it == pass_start)
|
||||
throw invalid_utf8(*it); // error - no lead byte in the sequence
|
||||
octet_iterator temp = it;
|
||||
return utf8::next(temp, end);
|
||||
}
|
||||
|
||||
template <typename octet_iterator, typename distance_type>
|
||||
void advance (octet_iterator& it, distance_type n, octet_iterator end)
|
||||
{
|
||||
for (distance_type i = 0; i < n; ++i)
|
||||
utf8::next(it, end);
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
typename std::iterator_traits<octet_iterator>::difference_type
|
||||
distance (octet_iterator first, octet_iterator last)
|
||||
{
|
||||
typename std::iterator_traits<octet_iterator>::difference_type dist;
|
||||
for (dist = 0; first < last; ++dist)
|
||||
utf8::next(first, last);
|
||||
return dist;
|
||||
}
|
||||
|
||||
template <typename u16bit_iterator, typename octet_iterator>
|
||||
octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result)
|
||||
{
|
||||
while (start != end) {
|
||||
uint32_t cp = utf8::internal::mask16(*start++);
|
||||
// Take care of surrogate pairs first
|
||||
if (utf8::internal::is_lead_surrogate(cp)) {
|
||||
if (start != end) {
|
||||
uint32_t trail_surrogate = utf8::internal::mask16(*start++);
|
||||
if (utf8::internal::is_trail_surrogate(trail_surrogate))
|
||||
cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;
|
||||
else
|
||||
throw invalid_utf16(static_cast<uint16_t>(trail_surrogate));
|
||||
}
|
||||
else
|
||||
throw invalid_utf16(static_cast<uint16_t>(cp));
|
||||
|
||||
}
|
||||
// Lone trail surrogate
|
||||
else if (utf8::internal::is_trail_surrogate(cp))
|
||||
throw invalid_utf16(static_cast<uint16_t>(cp));
|
||||
|
||||
result = utf8::append(cp, result);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
template <typename u16bit_iterator, typename octet_iterator>
|
||||
u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result)
|
||||
{
|
||||
while (start < end) {
|
||||
uint32_t cp = utf8::next(start, end);
|
||||
if (cp > 0xffff) { //make a surrogate pair
|
||||
*result++ = static_cast<uint16_t>((cp >> 10) + internal::LEAD_OFFSET);
|
||||
*result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
|
||||
}
|
||||
else
|
||||
*result++ = static_cast<uint16_t>(cp);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
template <typename octet_iterator, typename u32bit_iterator>
|
||||
octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result)
|
||||
{
|
||||
while (start != end)
|
||||
result = utf8::append(*(start++), result);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
template <typename octet_iterator, typename u32bit_iterator>
|
||||
u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result)
|
||||
{
|
||||
while (start < end)
|
||||
(*result++) = utf8::next(start, end);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// The iterator class
|
||||
template <typename octet_iterator>
|
||||
class iterator : public std::iterator <std::bidirectional_iterator_tag, uint32_t> {
|
||||
octet_iterator it;
|
||||
octet_iterator range_start;
|
||||
octet_iterator range_end;
|
||||
public:
|
||||
iterator () {}
|
||||
explicit iterator (const octet_iterator& octet_it,
|
||||
const octet_iterator& rangestart,
|
||||
const octet_iterator& rangeend) :
|
||||
it(octet_it), range_start(rangestart), range_end(rangeend)
|
||||
{
|
||||
if (it < range_start || it > range_end)
|
||||
throw std::out_of_range("Invalid utf-8 iterator position");
|
||||
}
|
||||
// the default "big three" are OK
|
||||
octet_iterator base () const { return it; }
|
||||
uint32_t operator * () const
|
||||
{
|
||||
octet_iterator temp = it;
|
||||
return utf8::next(temp, range_end);
|
||||
}
|
||||
bool operator == (const iterator& rhs) const
|
||||
{
|
||||
if (range_start != rhs.range_start || range_end != rhs.range_end)
|
||||
throw std::logic_error("Comparing utf-8 iterators defined with different ranges");
|
||||
return (it == rhs.it);
|
||||
}
|
||||
bool operator != (const iterator& rhs) const
|
||||
{
|
||||
return !(operator == (rhs));
|
||||
}
|
||||
iterator& operator ++ ()
|
||||
{
|
||||
utf8::next(it, range_end);
|
||||
return *this;
|
||||
}
|
||||
iterator operator ++ (int)
|
||||
{
|
||||
iterator temp = *this;
|
||||
utf8::next(it, range_end);
|
||||
return temp;
|
||||
}
|
||||
iterator& operator -- ()
|
||||
{
|
||||
utf8::prior(it, range_start);
|
||||
return *this;
|
||||
}
|
||||
iterator operator -- (int)
|
||||
{
|
||||
iterator temp = *this;
|
||||
utf8::prior(it, range_start);
|
||||
return temp;
|
||||
}
|
||||
}; // class iterator
|
||||
|
||||
} // namespace utf8
|
||||
|
||||
#endif //header guard
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,332 @@
|
|||
// Copyright 2006 Nemanja Trifunovic
|
||||
|
||||
/*
|
||||
Permission is hereby granted, free of charge, to any person or organization
|
||||
obtaining a copy of the software and accompanying documentation covered by
|
||||
this license (the "Software") to use, reproduce, display, distribute,
|
||||
execute, and transmit the Software, and to prepare derivative works of the
|
||||
Software, and to permit third-parties to whom the Software is furnished to
|
||||
do so, all subject to the following:
|
||||
|
||||
The copyright notices in the Software and this entire statement, including
|
||||
the above license grant, this restriction and the following disclaimer,
|
||||
must be included in all copies of the Software, in whole or in part, and
|
||||
all derivative works of the Software, unless such copies or derivative
|
||||
works are solely in the form of machine-executable object code generated by
|
||||
a source language processor.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
|
||||
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
|
||||
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
|
||||
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
|
||||
#ifndef UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||
#define UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||
|
||||
#include <iterator>
|
||||
|
||||
namespace utf8
|
||||
{
|
||||
// The typedefs for 8-bit, 16-bit and 32-bit unsigned integers
|
||||
// You may need to change them to match your system.
|
||||
// These typedefs have the same names as ones from cstdint, or boost/cstdint
|
||||
typedef unsigned char uint8_t;
|
||||
typedef unsigned short uint16_t;
|
||||
typedef unsigned int uint32_t;
|
||||
|
||||
// Helper code - not intended to be directly called by the library users. May be changed at any time
|
||||
namespace internal
|
||||
{
|
||||
// Unicode constants
|
||||
// Leading (high) surrogates: 0xd800 - 0xdbff
|
||||
// Trailing (low) surrogates: 0xdc00 - 0xdfff
|
||||
const uint16_t LEAD_SURROGATE_MIN = 0xd800u;
|
||||
const uint16_t LEAD_SURROGATE_MAX = 0xdbffu;
|
||||
const uint16_t TRAIL_SURROGATE_MIN = 0xdc00u;
|
||||
const uint16_t TRAIL_SURROGATE_MAX = 0xdfffu;
|
||||
const uint16_t LEAD_OFFSET = LEAD_SURROGATE_MIN - (0x10000 >> 10);
|
||||
const uint32_t SURROGATE_OFFSET = 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN;
|
||||
|
||||
// Maximum valid value for a Unicode code point
|
||||
const uint32_t CODE_POINT_MAX = 0x0010ffffu;
|
||||
|
||||
template<typename octet_type>
|
||||
inline uint8_t mask8(octet_type oc)
|
||||
{
|
||||
return static_cast<uint8_t>(0xff & oc);
|
||||
}
|
||||
template<typename u16_type>
|
||||
inline uint16_t mask16(u16_type oc)
|
||||
{
|
||||
return static_cast<uint16_t>(0xffff & oc);
|
||||
}
|
||||
template<typename octet_type>
|
||||
inline bool is_trail(octet_type oc)
|
||||
{
|
||||
return ((utf8::internal::mask8(oc) >> 6) == 0x2);
|
||||
}
|
||||
|
||||
template <typename u16>
|
||||
inline bool is_lead_surrogate(u16 cp)
|
||||
{
|
||||
return (cp >= LEAD_SURROGATE_MIN && cp <= LEAD_SURROGATE_MAX);
|
||||
}
|
||||
|
||||
template <typename u16>
|
||||
inline bool is_trail_surrogate(u16 cp)
|
||||
{
|
||||
return (cp >= TRAIL_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX);
|
||||
}
|
||||
|
||||
template <typename u16>
|
||||
inline bool is_surrogate(u16 cp)
|
||||
{
|
||||
return (cp >= LEAD_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX);
|
||||
}
|
||||
|
||||
template <typename u32>
|
||||
inline bool is_code_point_valid(u32 cp)
|
||||
{
|
||||
return (cp <= CODE_POINT_MAX && !utf8::internal::is_surrogate(cp));
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
inline typename std::iterator_traits<octet_iterator>::difference_type
|
||||
sequence_length(octet_iterator lead_it)
|
||||
{
|
||||
uint8_t lead = utf8::internal::mask8(*lead_it);
|
||||
if (lead < 0x80)
|
||||
return 1;
|
||||
else if ((lead >> 5) == 0x6)
|
||||
return 2;
|
||||
else if ((lead >> 4) == 0xe)
|
||||
return 3;
|
||||
else if ((lead >> 3) == 0x1e)
|
||||
return 4;
|
||||
else
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <typename octet_difference_type>
|
||||
inline bool is_overlong_sequence(uint32_t cp, octet_difference_type length)
|
||||
{
|
||||
if (cp < 0x80) {
|
||||
if (length != 1)
|
||||
return true;
|
||||
}
|
||||
else if (cp < 0x800) {
|
||||
if (length != 2)
|
||||
return true;
|
||||
}
|
||||
else if (cp < 0x10000) {
|
||||
if (length != 3)
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
enum utf_error {UTF8_OK, NOT_ENOUGH_ROOM, INVALID_LEAD, INCOMPLETE_SEQUENCE, OVERLONG_SEQUENCE, INVALID_CODE_POINT};
|
||||
|
||||
/// Helper for get_sequence_x
|
||||
template <typename octet_iterator>
|
||||
utf_error increase_safely(octet_iterator& it, octet_iterator end)
|
||||
{
|
||||
if (++it == end)
|
||||
return NOT_ENOUGH_ROOM;
|
||||
|
||||
if (!utf8::internal::is_trail(*it))
|
||||
return INCOMPLETE_SEQUENCE;
|
||||
|
||||
return UTF8_OK;
|
||||
}
|
||||
|
||||
#define UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(IT, END) {utf_error ret = increase_safely(IT, END); if (ret != UTF8_OK) return ret;}
|
||||
|
||||
/// get_sequence_x functions decode utf-8 sequences of the length x
|
||||
template <typename octet_iterator>
|
||||
utf_error get_sequence_1(octet_iterator& it, octet_iterator end, uint32_t& code_point)
|
||||
{
|
||||
if (it == end)
|
||||
return NOT_ENOUGH_ROOM;
|
||||
|
||||
code_point = utf8::internal::mask8(*it);
|
||||
|
||||
return UTF8_OK;
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
utf_error get_sequence_2(octet_iterator& it, octet_iterator end, uint32_t& code_point)
|
||||
{
|
||||
if (it == end)
|
||||
return NOT_ENOUGH_ROOM;
|
||||
|
||||
code_point = utf8::internal::mask8(*it);
|
||||
|
||||
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
|
||||
|
||||
code_point = ((code_point << 6) & 0x7ff) + ((*it) & 0x3f);
|
||||
|
||||
return UTF8_OK;
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
utf_error get_sequence_3(octet_iterator& it, octet_iterator end, uint32_t& code_point)
|
||||
{
|
||||
if (it == end)
|
||||
return NOT_ENOUGH_ROOM;
|
||||
|
||||
code_point = utf8::internal::mask8(*it);
|
||||
|
||||
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
|
||||
|
||||
code_point = ((code_point << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff);
|
||||
|
||||
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
|
||||
|
||||
code_point += (*it) & 0x3f;
|
||||
|
||||
return UTF8_OK;
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t& code_point)
|
||||
{
|
||||
if (it == end)
|
||||
return NOT_ENOUGH_ROOM;
|
||||
|
||||
code_point = utf8::internal::mask8(*it);
|
||||
|
||||
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
|
||||
|
||||
code_point = ((code_point << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff);
|
||||
|
||||
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
|
||||
|
||||
code_point += (utf8::internal::mask8(*it) << 6) & 0xfff;
|
||||
|
||||
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
|
||||
|
||||
code_point += (*it) & 0x3f;
|
||||
|
||||
return UTF8_OK;
|
||||
}
|
||||
|
||||
#undef UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR
|
||||
|
||||
template <typename octet_iterator>
|
||||
utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t& code_point)
|
||||
{
|
||||
if (it == end)
|
||||
return NOT_ENOUGH_ROOM;
|
||||
|
||||
// Save the original value of it so we can go back in case of failure
|
||||
// Of course, it does not make much sense with i.e. stream iterators
|
||||
octet_iterator original_it = it;
|
||||
|
||||
uint32_t cp = 0;
|
||||
// Determine the sequence length based on the lead octet
|
||||
typedef typename std::iterator_traits<octet_iterator>::difference_type octet_difference_type;
|
||||
const octet_difference_type length = utf8::internal::sequence_length(it);
|
||||
|
||||
// Get trail octets and calculate the code point
|
||||
utf_error err = UTF8_OK;
|
||||
switch (length) {
|
||||
case 0:
|
||||
return INVALID_LEAD;
|
||||
case 1:
|
||||
err = utf8::internal::get_sequence_1(it, end, cp);
|
||||
break;
|
||||
case 2:
|
||||
err = utf8::internal::get_sequence_2(it, end, cp);
|
||||
break;
|
||||
case 3:
|
||||
err = utf8::internal::get_sequence_3(it, end, cp);
|
||||
break;
|
||||
case 4:
|
||||
err = utf8::internal::get_sequence_4(it, end, cp);
|
||||
break;
|
||||
}
|
||||
|
||||
if (err == UTF8_OK) {
|
||||
// Decoding succeeded. Now, security checks...
|
||||
if (utf8::internal::is_code_point_valid(cp)) {
|
||||
if (!utf8::internal::is_overlong_sequence(cp, length)){
|
||||
// Passed! Return here.
|
||||
code_point = cp;
|
||||
++it;
|
||||
return UTF8_OK;
|
||||
}
|
||||
else
|
||||
err = OVERLONG_SEQUENCE;
|
||||
}
|
||||
else
|
||||
err = INVALID_CODE_POINT;
|
||||
}
|
||||
|
||||
// Failure branch - restore the original value of the iterator
|
||||
it = original_it;
|
||||
return err;
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
inline utf_error validate_next(octet_iterator& it, octet_iterator end) {
|
||||
uint32_t ignored;
|
||||
return utf8::internal::validate_next(it, end, ignored);
|
||||
}
|
||||
|
||||
} // namespace internal
|
||||
|
||||
/// The library API - functions intended to be called by the users
|
||||
|
||||
// Byte order mark
|
||||
const uint8_t bom[] = {0xef, 0xbb, 0xbf};
|
||||
|
||||
template <typename octet_iterator>
|
||||
octet_iterator find_invalid(octet_iterator start, octet_iterator end)
|
||||
{
|
||||
octet_iterator result = start;
|
||||
while (result != end) {
|
||||
utf8::internal::utf_error err_code = utf8::internal::validate_next(result, end);
|
||||
if (err_code != internal::UTF8_OK)
|
||||
return result;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
inline bool is_valid(octet_iterator start, octet_iterator end)
|
||||
{
|
||||
return (utf8::find_invalid(start, end) == end);
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
inline bool starts_with_bom (octet_iterator it, octet_iterator end)
|
||||
{
|
||||
return (
|
||||
((it != end) && (utf8::internal::mask8(*it++)) == bom[0]) &&
|
||||
((it != end) && (utf8::internal::mask8(*it++)) == bom[1]) &&
|
||||
((it != end) && (utf8::internal::mask8(*it)) == bom[2])
|
||||
);
|
||||
}
|
||||
|
||||
//Deprecated in release 2.3
|
||||
template <typename octet_iterator>
|
||||
inline bool is_bom (octet_iterator it)
|
||||
{
|
||||
return (
|
||||
(utf8::internal::mask8(*it++)) == bom[0] &&
|
||||
(utf8::internal::mask8(*it++)) == bom[1] &&
|
||||
(utf8::internal::mask8(*it)) == bom[2]
|
||||
);
|
||||
}
|
||||
} // namespace utf8
|
||||
|
||||
#endif // header guard
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,228 @@
|
|||
// Copyright 2006 Nemanja Trifunovic
|
||||
|
||||
/*
|
||||
Permission is hereby granted, free of charge, to any person or organization
|
||||
obtaining a copy of the software and accompanying documentation covered by
|
||||
this license (the "Software") to use, reproduce, display, distribute,
|
||||
execute, and transmit the Software, and to prepare derivative works of the
|
||||
Software, and to permit third-parties to whom the Software is furnished to
|
||||
do so, all subject to the following:
|
||||
|
||||
The copyright notices in the Software and this entire statement, including
|
||||
the above license grant, this restriction and the following disclaimer,
|
||||
must be included in all copies of the Software, in whole or in part, and
|
||||
all derivative works of the Software, unless such copies or derivative
|
||||
works are solely in the form of machine-executable object code generated by
|
||||
a source language processor.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
|
||||
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
|
||||
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
|
||||
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
|
||||
#ifndef UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||
#define UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||
|
||||
#include "core.h"
|
||||
|
||||
namespace utf8
|
||||
{
|
||||
namespace unchecked
|
||||
{
|
||||
template <typename octet_iterator>
|
||||
octet_iterator append(uint32_t cp, octet_iterator result)
|
||||
{
|
||||
if (cp < 0x80) // one octet
|
||||
*(result++) = static_cast<uint8_t>(cp);
|
||||
else if (cp < 0x800) { // two octets
|
||||
*(result++) = static_cast<uint8_t>((cp >> 6) | 0xc0);
|
||||
*(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
|
||||
}
|
||||
else if (cp < 0x10000) { // three octets
|
||||
*(result++) = static_cast<uint8_t>((cp >> 12) | 0xe0);
|
||||
*(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80);
|
||||
*(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
|
||||
}
|
||||
else { // four octets
|
||||
*(result++) = static_cast<uint8_t>((cp >> 18) | 0xf0);
|
||||
*(result++) = static_cast<uint8_t>(((cp >> 12) & 0x3f)| 0x80);
|
||||
*(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80);
|
||||
*(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
uint32_t next(octet_iterator& it)
|
||||
{
|
||||
uint32_t cp = utf8::internal::mask8(*it);
|
||||
typename std::iterator_traits<octet_iterator>::difference_type length = utf8::internal::sequence_length(it);
|
||||
switch (length) {
|
||||
case 1:
|
||||
break;
|
||||
case 2:
|
||||
it++;
|
||||
cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f);
|
||||
break;
|
||||
case 3:
|
||||
++it;
|
||||
cp = ((cp << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff);
|
||||
++it;
|
||||
cp += (*it) & 0x3f;
|
||||
break;
|
||||
case 4:
|
||||
++it;
|
||||
cp = ((cp << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff);
|
||||
++it;
|
||||
cp += (utf8::internal::mask8(*it) << 6) & 0xfff;
|
||||
++it;
|
||||
cp += (*it) & 0x3f;
|
||||
break;
|
||||
}
|
||||
++it;
|
||||
return cp;
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
uint32_t peek_next(octet_iterator it)
|
||||
{
|
||||
return utf8::unchecked::next(it);
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
uint32_t prior(octet_iterator& it)
|
||||
{
|
||||
while (utf8::internal::is_trail(*(--it))) ;
|
||||
octet_iterator temp = it;
|
||||
return utf8::unchecked::next(temp);
|
||||
}
|
||||
|
||||
// Deprecated in versions that include prior, but only for the sake of consistency (see utf8::previous)
|
||||
template <typename octet_iterator>
|
||||
inline uint32_t previous(octet_iterator& it)
|
||||
{
|
||||
return utf8::unchecked::prior(it);
|
||||
}
|
||||
|
||||
template <typename octet_iterator, typename distance_type>
|
||||
void advance (octet_iterator& it, distance_type n)
|
||||
{
|
||||
for (distance_type i = 0; i < n; ++i)
|
||||
utf8::unchecked::next(it);
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
typename std::iterator_traits<octet_iterator>::difference_type
|
||||
distance (octet_iterator first, octet_iterator last)
|
||||
{
|
||||
typename std::iterator_traits<octet_iterator>::difference_type dist;
|
||||
for (dist = 0; first < last; ++dist)
|
||||
utf8::unchecked::next(first);
|
||||
return dist;
|
||||
}
|
||||
|
||||
template <typename u16bit_iterator, typename octet_iterator>
|
||||
octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result)
|
||||
{
|
||||
while (start != end) {
|
||||
uint32_t cp = utf8::internal::mask16(*start++);
|
||||
// Take care of surrogate pairs first
|
||||
if (utf8::internal::is_lead_surrogate(cp)) {
|
||||
uint32_t trail_surrogate = utf8::internal::mask16(*start++);
|
||||
cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;
|
||||
}
|
||||
result = utf8::unchecked::append(cp, result);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
template <typename u16bit_iterator, typename octet_iterator>
|
||||
u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result)
|
||||
{
|
||||
while (start < end) {
|
||||
uint32_t cp = utf8::unchecked::next(start);
|
||||
if (cp > 0xffff) { //make a surrogate pair
|
||||
*result++ = static_cast<uint16_t>((cp >> 10) + internal::LEAD_OFFSET);
|
||||
*result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
|
||||
}
|
||||
else
|
||||
*result++ = static_cast<uint16_t>(cp);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
template <typename octet_iterator, typename u32bit_iterator>
|
||||
octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result)
|
||||
{
|
||||
while (start != end)
|
||||
result = utf8::unchecked::append(*(start++), result);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
template <typename octet_iterator, typename u32bit_iterator>
|
||||
u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result)
|
||||
{
|
||||
while (start < end)
|
||||
(*result++) = utf8::unchecked::next(start);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// The iterator class
|
||||
template <typename octet_iterator>
|
||||
class iterator : public std::iterator <std::bidirectional_iterator_tag, uint32_t> {
|
||||
octet_iterator it;
|
||||
public:
|
||||
iterator () {}
|
||||
explicit iterator (const octet_iterator& octet_it): it(octet_it) {}
|
||||
// the default "big three" are OK
|
||||
octet_iterator base () const { return it; }
|
||||
uint32_t operator * () const
|
||||
{
|
||||
octet_iterator temp = it;
|
||||
return utf8::unchecked::next(temp);
|
||||
}
|
||||
bool operator == (const iterator& rhs) const
|
||||
{
|
||||
return (it == rhs.it);
|
||||
}
|
||||
bool operator != (const iterator& rhs) const
|
||||
{
|
||||
return !(operator == (rhs));
|
||||
}
|
||||
iterator& operator ++ ()
|
||||
{
|
||||
::std::advance(it, utf8::internal::sequence_length(it));
|
||||
return *this;
|
||||
}
|
||||
iterator operator ++ (int)
|
||||
{
|
||||
iterator temp = *this;
|
||||
::std::advance(it, utf8::internal::sequence_length(it));
|
||||
return temp;
|
||||
}
|
||||
iterator& operator -- ()
|
||||
{
|
||||
utf8::unchecked::prior(it);
|
||||
return *this;
|
||||
}
|
||||
iterator operator -- (int)
|
||||
{
|
||||
iterator temp = *this;
|
||||
utf8::unchecked::prior(it);
|
||||
return temp;
|
||||
}
|
||||
}; // class iterator
|
||||
|
||||
} // namespace utf8::unchecked
|
||||
} // namespace utf8
|
||||
|
||||
|
||||
#endif // header guard
|
||||
|
||||
Binary file not shown.
|
|
@ -0,0 +1,212 @@
|
|||
|
||||
UTF-8 encoded sample plain-text file
|
||||
‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾
|
||||
|
||||
Markus Kuhn [ˈmaʳkʊs kuːn] <http://www.cl.cam.ac.uk/~mgk25/> — 2002-07-25
|
||||
|
||||
|
||||
The ASCII compatible UTF-8 encoding used in this plain-text file
|
||||
is defined in Unicode, ISO 10646-1, and RFC 2279.
|
||||
|
||||
|
||||
Using Unicode/UTF-8, you can write in emails and source code things such as
|
||||
|
||||
Mathematics and sciences:
|
||||
|
||||
∮ E⋅da = Q, n → ∞, ∑ f(i) = ∏ g(i), ⎧⎡⎛┌─────┐⎞⎤⎫
|
||||
⎪⎢⎜│a²+b³ ⎟⎥⎪
|
||||
∀x∈ℝ: ⌈x⌉ = −⌊−x⌋, α ∧ ¬β = ¬(¬α ∨ β), ⎪⎢⎜│───── ⎟⎥⎪
|
||||
⎪⎢⎜⎷ c₈ ⎟⎥⎪
|
||||
ℕ ⊆ ℕ₀ ⊂ ℤ ⊂ ℚ ⊂ ℝ ⊂ ℂ, ⎨⎢⎜ ⎟⎥⎬
|
||||
⎪⎢⎜ ∞ ⎟⎥⎪
|
||||
⊥ < a ≠ b ≡ c ≤ d ≪ ⊤ ⇒ (⟦A⟧ ⇔ ⟪B⟫), ⎪⎢⎜ ⎲ ⎟⎥⎪
|
||||
⎪⎢⎜ ⎳aⁱ-bⁱ⎟⎥⎪
|
||||
2H₂ + O₂ ⇌ 2H₂O, R = 4.7 kΩ, ⌀ 200 mm ⎩⎣⎝i=1 ⎠⎦⎭
|
||||
|
||||
Linguistics and dictionaries:
|
||||
|
||||
ði ıntəˈnæʃənəl fəˈnɛtık əsoʊsiˈeıʃn
|
||||
Y [ˈʏpsilɔn], Yen [jɛn], Yoga [ˈjoːgɑ]
|
||||
|
||||
APL:
|
||||
|
||||
((V⍳V)=⍳⍴V)/V←,V ⌷←⍳→⍴∆∇⊃‾⍎⍕⌈
|
||||
|
||||
Nicer typography in plain text files:
|
||||
|
||||
╔══════════════════════════════════════════╗
|
||||
║ ║
|
||||
║ • ‘single’ and “double” quotes ║
|
||||
║ ║
|
||||
║ • Curly apostrophes: “We’ve been here” ║
|
||||
║ ║
|
||||
║ • Latin-1 apostrophe and accents: '´` ║
|
||||
║ ║
|
||||
║ • ‚deutsche‘ „Anführungszeichen“ ║
|
||||
║ ║
|
||||
║ • †, ‡, ‰, •, 3–4, —, −5/+5, ™, … ║
|
||||
║ ║
|
||||
║ • ASCII safety test: 1lI|, 0OD, 8B ║
|
||||
║ ╭─────────╮ ║
|
||||
║ • the euro symbol: │ 14.95 € │ ║
|
||||
║ ╰─────────╯ ║
|
||||
╚══════════════════════════════════════════╝
|
||||
|
||||
Combining characters:
|
||||
|
||||
STARGΛ̊TE SG-1, a = v̇ = r̈, a⃑ ⊥ b⃑
|
||||
|
||||
Greek (in Polytonic):
|
||||
|
||||
The Greek anthem:
|
||||
|
||||
Σὲ γνωρίζω ἀπὸ τὴν κόψη
|
||||
τοῦ σπαθιοῦ τὴν τρομερή,
|
||||
σὲ γνωρίζω ἀπὸ τὴν ὄψη
|
||||
ποὺ μὲ βία μετράει τὴ γῆ.
|
||||
|
||||
᾿Απ᾿ τὰ κόκκαλα βγαλμένη
|
||||
τῶν ῾Ελλήνων τὰ ἱερά
|
||||
καὶ σὰν πρῶτα ἀνδρειωμένη
|
||||
χαῖρε, ὦ χαῖρε, ᾿Ελευθεριά!
|
||||
|
||||
From a speech of Demosthenes in the 4th century BC:
|
||||
|
||||
Οὐχὶ ταὐτὰ παρίσταταί μοι γιγνώσκειν, ὦ ἄνδρες ᾿Αθηναῖοι,
|
||||
ὅταν τ᾿ εἰς τὰ πράγματα ἀποβλέψω καὶ ὅταν πρὸς τοὺς
|
||||
λόγους οὓς ἀκούω· τοὺς μὲν γὰρ λόγους περὶ τοῦ
|
||||
τιμωρήσασθαι Φίλιππον ὁρῶ γιγνομένους, τὰ δὲ πράγματ᾿
|
||||
εἰς τοῦτο προήκοντα, ὥσθ᾿ ὅπως μὴ πεισόμεθ᾿ αὐτοὶ
|
||||
πρότερον κακῶς σκέψασθαι δέον. οὐδέν οὖν ἄλλο μοι δοκοῦσιν
|
||||
οἱ τὰ τοιαῦτα λέγοντες ἢ τὴν ὑπόθεσιν, περὶ ἧς βουλεύεσθαι,
|
||||
οὐχὶ τὴν οὖσαν παριστάντες ὑμῖν ἁμαρτάνειν. ἐγὼ δέ, ὅτι μέν
|
||||
ποτ᾿ ἐξῆν τῇ πόλει καὶ τὰ αὑτῆς ἔχειν ἀσφαλῶς καὶ Φίλιππον
|
||||
τιμωρήσασθαι, καὶ μάλ᾿ ἀκριβῶς οἶδα· ἐπ᾿ ἐμοῦ γάρ, οὐ πάλαι
|
||||
γέγονεν ταῦτ᾿ ἀμφότερα· νῦν μέντοι πέπεισμαι τοῦθ᾿ ἱκανὸν
|
||||
προλαβεῖν ἡμῖν εἶναι τὴν πρώτην, ὅπως τοὺς συμμάχους
|
||||
σώσομεν. ἐὰν γὰρ τοῦτο βεβαίως ὑπάρξῃ, τότε καὶ περὶ τοῦ
|
||||
τίνα τιμωρήσεταί τις καὶ ὃν τρόπον ἐξέσται σκοπεῖν· πρὶν δὲ
|
||||
τὴν ἀρχὴν ὀρθῶς ὑποθέσθαι, μάταιον ἡγοῦμαι περὶ τῆς
|
||||
τελευτῆς ὁντινοῦν ποιεῖσθαι λόγον.
|
||||
|
||||
Δημοσθένους, Γ´ ᾿Ολυνθιακὸς
|
||||
|
||||
Georgian:
|
||||
|
||||
From a Unicode conference invitation:
|
||||
|
||||
გთხოვთ ახლავე გაიაროთ რეგისტრაცია Unicode-ის მეათე საერთაშორისო
|
||||
კონფერენციაზე დასასწრებად, რომელიც გაიმართება 10-12 მარტს,
|
||||
ქ. მაინცში, გერმანიაში. კონფერენცია შეჰკრებს ერთად მსოფლიოს
|
||||
ექსპერტებს ისეთ დარგებში როგორიცაა ინტერნეტი და Unicode-ი,
|
||||
ინტერნაციონალიზაცია და ლოკალიზაცია, Unicode-ის გამოყენება
|
||||
ოპერაციულ სისტემებსა, და გამოყენებით პროგრამებში, შრიფტებში,
|
||||
ტექსტების დამუშავებასა და მრავალენოვან კომპიუტერულ სისტემებში.
|
||||
|
||||
Russian:
|
||||
|
||||
From a Unicode conference invitation:
|
||||
|
||||
Зарегистрируйтесь сейчас на Десятую Международную Конференцию по
|
||||
Unicode, которая состоится 10-12 марта 1997 года в Майнце в Германии.
|
||||
Конференция соберет широкий круг экспертов по вопросам глобального
|
||||
Интернета и Unicode, локализации и интернационализации, воплощению и
|
||||
применению Unicode в различных операционных системах и программных
|
||||
приложениях, шрифтах, верстке и многоязычных компьютерных системах.
|
||||
|
||||
Thai (UCS Level 2):
|
||||
|
||||
Excerpt from a poetry on The Romance of The Three Kingdoms (a Chinese
|
||||
classic 'San Gua'):
|
||||
|
||||
[----------------------------|------------------------]
|
||||
๏ แผ่นดินฮั่นเสื่อมโทรมแสนสังเวช พระปกเกศกองบู๊กู้ขึ้นใหม่
|
||||
สิบสองกษัตริย์ก่อนหน้าแลถัดไป สององค์ไซร้โง่เขลาเบาปัญญา
|
||||
ทรงนับถือขันทีเป็นที่พึ่ง บ้านเมืองจึงวิปริตเป็นนักหนา
|
||||
โฮจิ๋นเรียกทัพทั่วหัวเมืองมา หมายจะฆ่ามดชั่วตัวสำคัญ
|
||||
เหมือนขับไสไล่เสือจากเคหา รับหมาป่าเข้ามาเลยอาสัญ
|
||||
ฝ่ายอ้องอุ้นยุแยกให้แตกกัน ใช้สาวนั้นเป็นชนวนชื่นชวนใจ
|
||||
พลันลิฉุยกุยกีกลับก่อเหตุ ช่างอาเพศจริงหนาฟ้าร้องไห้
|
||||
ต้องรบราฆ่าฟันจนบรรลัย ฤๅหาใครค้ำชูกู้บรรลังก์ ฯ
|
||||
|
||||
(The above is a two-column text. If combining characters are handled
|
||||
correctly, the lines of the second column should be aligned with the
|
||||
| character above.)
|
||||
|
||||
Ethiopian:
|
||||
|
||||
Proverbs in the Amharic language:
|
||||
|
||||
ሰማይ አይታረስ ንጉሥ አይከሰስ።
|
||||
ብላ ካለኝ እንደአባቴ በቆመጠኝ።
|
||||
ጌጥ ያለቤቱ ቁምጥና ነው።
|
||||
ደሀ በሕልሙ ቅቤ ባይጠጣ ንጣት በገደለው።
|
||||
የአፍ ወለምታ በቅቤ አይታሽም።
|
||||
አይጥ በበላ ዳዋ ተመታ።
|
||||
ሲተረጉሙ ይደረግሙ።
|
||||
ቀስ በቀስ፥ ዕንቁላል በእግሩ ይሄዳል።
|
||||
ድር ቢያብር አንበሳ ያስር።
|
||||
ሰው እንደቤቱ እንጅ እንደ ጉረቤቱ አይተዳደርም።
|
||||
እግዜር የከፈተውን ጉሮሮ ሳይዘጋው አይድርም።
|
||||
የጎረቤት ሌባ፥ ቢያዩት ይስቅ ባያዩት ያጠልቅ።
|
||||
ሥራ ከመፍታት ልጄን ላፋታት።
|
||||
ዓባይ ማደሪያ የለው፥ ግንድ ይዞ ይዞራል።
|
||||
የእስላም አገሩ መካ የአሞራ አገሩ ዋርካ።
|
||||
ተንጋሎ ቢተፉ ተመልሶ ባፉ።
|
||||
ወዳጅህ ማር ቢሆን ጨርስህ አትላሰው።
|
||||
እግርህን በፍራሽህ ልክ ዘርጋ።
|
||||
|
||||
Runes:
|
||||
|
||||
ᚻᛖ ᚳᚹᚫᚦ ᚦᚫᛏ ᚻᛖ ᛒᚢᛞᛖ ᚩᚾ ᚦᚫᛗ ᛚᚪᚾᛞᛖ ᚾᚩᚱᚦᚹᛖᚪᚱᛞᚢᛗ ᚹᛁᚦ ᚦᚪ ᚹᛖᛥᚫ
|
||||
|
||||
(Old English, which transcribed into Latin reads 'He cwaeth that he
|
||||
bude thaem lande northweardum with tha Westsae.' and means 'He said
|
||||
that he lived in the northern land near the Western Sea.')
|
||||
|
||||
Braille:
|
||||
|
||||
⡌⠁⠧⠑ ⠼⠁⠒ ⡍⠜⠇⠑⠹⠰⠎ ⡣⠕⠌
|
||||
|
||||
⡍⠜⠇⠑⠹ ⠺⠁⠎ ⠙⠑⠁⠙⠒ ⠞⠕ ⠃⠑⠛⠔ ⠺⠊⠹⠲ ⡹⠻⠑ ⠊⠎ ⠝⠕ ⠙⠳⠃⠞
|
||||
⠱⠁⠞⠑⠧⠻ ⠁⠃⠳⠞ ⠹⠁⠞⠲ ⡹⠑ ⠗⠑⠛⠊⠌⠻ ⠕⠋ ⠙⠊⠎ ⠃⠥⠗⠊⠁⠇ ⠺⠁⠎
|
||||
⠎⠊⠛⠝⠫ ⠃⠹ ⠹⠑ ⠊⠇⠻⠛⠹⠍⠁⠝⠂ ⠹⠑ ⠊⠇⠻⠅⠂ ⠹⠑ ⠥⠝⠙⠻⠞⠁⠅⠻⠂
|
||||
⠁⠝⠙ ⠹⠑ ⠡⠊⠑⠋ ⠍⠳⠗⠝⠻⠲ ⡎⠊⠗⠕⠕⠛⠑ ⠎⠊⠛⠝⠫ ⠊⠞⠲ ⡁⠝⠙
|
||||
⡎⠊⠗⠕⠕⠛⠑⠰⠎ ⠝⠁⠍⠑ ⠺⠁⠎ ⠛⠕⠕⠙ ⠥⠏⠕⠝ ⠰⡡⠁⠝⠛⠑⠂ ⠋⠕⠗ ⠁⠝⠹⠹⠔⠛ ⠙⠑
|
||||
⠡⠕⠎⠑ ⠞⠕ ⠏⠥⠞ ⠙⠊⠎ ⠙⠁⠝⠙ ⠞⠕⠲
|
||||
|
||||
⡕⠇⠙ ⡍⠜⠇⠑⠹ ⠺⠁⠎ ⠁⠎ ⠙⠑⠁⠙ ⠁⠎ ⠁ ⠙⠕⠕⠗⠤⠝⠁⠊⠇⠲
|
||||
|
||||
⡍⠔⠙⠖ ⡊ ⠙⠕⠝⠰⠞ ⠍⠑⠁⠝ ⠞⠕ ⠎⠁⠹ ⠹⠁⠞ ⡊ ⠅⠝⠪⠂ ⠕⠋ ⠍⠹
|
||||
⠪⠝ ⠅⠝⠪⠇⠫⠛⠑⠂ ⠱⠁⠞ ⠹⠻⠑ ⠊⠎ ⠏⠜⠞⠊⠊⠥⠇⠜⠇⠹ ⠙⠑⠁⠙ ⠁⠃⠳⠞
|
||||
⠁ ⠙⠕⠕⠗⠤⠝⠁⠊⠇⠲ ⡊ ⠍⠊⠣⠞ ⠙⠁⠧⠑ ⠃⠑⠲ ⠔⠊⠇⠔⠫⠂ ⠍⠹⠎⠑⠇⠋⠂ ⠞⠕
|
||||
⠗⠑⠛⠜⠙ ⠁ ⠊⠕⠋⠋⠔⠤⠝⠁⠊⠇ ⠁⠎ ⠹⠑ ⠙⠑⠁⠙⠑⠌ ⠏⠊⠑⠊⠑ ⠕⠋ ⠊⠗⠕⠝⠍⠕⠝⠛⠻⠹
|
||||
⠔ ⠹⠑ ⠞⠗⠁⠙⠑⠲ ⡃⠥⠞ ⠹⠑ ⠺⠊⠎⠙⠕⠍ ⠕⠋ ⠳⠗ ⠁⠝⠊⠑⠌⠕⠗⠎
|
||||
⠊⠎ ⠔ ⠹⠑ ⠎⠊⠍⠊⠇⠑⠆ ⠁⠝⠙ ⠍⠹ ⠥⠝⠙⠁⠇⠇⠪⠫ ⠙⠁⠝⠙⠎
|
||||
⠩⠁⠇⠇ ⠝⠕⠞ ⠙⠊⠌⠥⠗⠃ ⠊⠞⠂ ⠕⠗ ⠹⠑ ⡊⠳⠝⠞⠗⠹⠰⠎ ⠙⠕⠝⠑ ⠋⠕⠗⠲ ⡹⠳
|
||||
⠺⠊⠇⠇ ⠹⠻⠑⠋⠕⠗⠑ ⠏⠻⠍⠊⠞ ⠍⠑ ⠞⠕ ⠗⠑⠏⠑⠁⠞⠂ ⠑⠍⠏⠙⠁⠞⠊⠊⠁⠇⠇⠹⠂ ⠹⠁⠞
|
||||
⡍⠜⠇⠑⠹ ⠺⠁⠎ ⠁⠎ ⠙⠑⠁⠙ ⠁⠎ ⠁ ⠙⠕⠕⠗⠤⠝⠁⠊⠇⠲
|
||||
|
||||
(The first couple of paragraphs of "A Christmas Carol" by Dickens)
|
||||
|
||||
Compact font selection example text:
|
||||
|
||||
ABCDEFGHIJKLMNOPQRSTUVWXYZ /0123456789
|
||||
abcdefghijklmnopqrstuvwxyz £©µÀÆÖÞßéöÿ
|
||||
–—‘“”„†•…‰™œŠŸž€ ΑΒΓΔΩαβγδω АБВГДабвгд
|
||||
∀∂∈ℝ∧∪≡∞ ↑↗↨↻⇣ ┐┼╔╘░►☺♀ fi<>⑀₂ἠḂӥẄɐː⍎אԱა
|
||||
|
||||
Greetings in various languages:
|
||||
|
||||
Hello world, Καλημέρα κόσμε, コンニチハ
|
||||
|
||||
Box drawing alignment tests: █
|
||||
▉
|
||||
╔══╦══╗ ┌──┬──┐ ╭──┬──╮ ╭──┬──╮ ┏━━┳━━┓ ┎┒┏┑ ╷ ╻ ┏┯┓ ┌┰┐ ▊ ╱╲╱╲╳╳╳
|
||||
║┌─╨─┐║ │╔═╧═╗│ │╒═╪═╕│ │╓─╁─╖│ ┃┌─╂─┐┃ ┗╃╄┙ ╶┼╴╺╋╸┠┼┨ ┝╋┥ ▋ ╲╱╲╱╳╳╳
|
||||
║│╲ ╱│║ │║ ║│ ││ │ ││ │║ ┃ ║│ ┃│ ╿ │┃ ┍╅╆┓ ╵ ╹ ┗┷┛ └┸┘ ▌ ╱╲╱╲╳╳╳
|
||||
╠╡ ╳ ╞╣ ├╢ ╟┤ ├┼─┼─┼┤ ├╫─╂─╫┤ ┣┿╾┼╼┿┫ ┕┛┖┚ ┌┄┄┐ ╎ ┏┅┅┓ ┋ ▍ ╲╱╲╱╳╳╳
|
||||
║│╱ ╲│║ │║ ║│ ││ │ ││ │║ ┃ ║│ ┃│ ╽ │┃ ░░▒▒▓▓██ ┊ ┆ ╎ ╏ ┇ ┋ ▎
|
||||
║└─╥─┘║ │╚═╤═╝│ │╘═╪═╛│ │╙─╀─╜│ ┃└─╂─┘┃ ░░▒▒▓▓██ ┊ ┆ ╎ ╏ ┇ ┋ ▏
|
||||
╚══╩══╝ └──┴──┘ ╰──┴──╯ ╰──┴──╯ ┗━━┻━━┛ ▗▄▖▛▀▜ └╌╌┘ ╎ ┗╍╍┛ ┋ ▁▂▃▄▅▆▇█
|
||||
▝▀▘▙▄▟
|
||||
|
|
@ -0,0 +1,167 @@
|
|||
? *Unicode Transcriptions* Notes <#Notes>
|
||||
|
||||
Glyphs <http://www.macchiato.com/unicode/show.html> | Samples
|
||||
<http://www.macchiato.com/unicode/Unicode_transcriptions.html> | Charts
|
||||
<http://www.macchiato.com/unicode/charts.html> | UTF
|
||||
<http://www.macchiato.com/unicode/convert.html> | Forms
|
||||
<http://www-4.ibm.com/software/developer/library/utfencodingforms/> |
|
||||
Home <http://www.macchiato.com>.
|
||||
<http://member.linkexchange.com/cgi-bin/fc/fastcounter-login?750641>
|
||||
|
||||
Name Text Image
|
||||
Arabic (Arabic) يونِكود ?
|
||||
Arabic (Persian) یونیکُد / ?/
|
||||
Armenian Յունիկօդ
|
||||
Bengali য়ূনিকোড
|
||||
Bopomofo ㄊㄨㄥ˅ ㄧˋ ㄇㄚ˅
|
||||
ㄨㄢˋ ㄍㄨㄛˊ ㄇㄚ˅
|
||||
Braille
|
||||
Buhid
|
||||
Canadian Aboriginal ᔫᗂᑰᑦ
|
||||
Cherokee ᏳᏂᎪᏛ
|
||||
Cypriot
|
||||
Cyrillic (Russian) Юникод ?
|
||||
Deseret (English) ???????
|
||||
Devanagari (Hindi) यूनिकोड ?
|
||||
Ethiopic ዩኒኮድ
|
||||
Georgian უნიკოდი ?
|
||||
Gothic
|
||||
Greek Γιούνικοντ
|
||||
Gujarati યૂનિકોડ
|
||||
Gurmukhi ਯੂਨਿਕੋਡ
|
||||
Han (Chinese) 统一码 ?
|
||||
統一碼 ?
|
||||
万国码 ?
|
||||
萬國碼 ?
|
||||
Hangul 유니코드
|
||||
Hanunoo
|
||||
Hebrew יוניקוד
|
||||
Hebrew (pointed) יוּנִיקוׁד
|
||||
Hebrew (Yiddish) יוניקאָד ?
|
||||
Hiragana (Japanese) ゆにこおど
|
||||
Katakana (Japanese) ユニコード ?
|
||||
Kannada ಯೂನಿಕೋಡ್
|
||||
Khmer យូនីគោដ
|
||||
Lao
|
||||
Latin Unicode Unicode
|
||||
Latin (IPA <#English_Pronunciation>) ˈjunɪˌkoːd ?
|
||||
Latin (Am. Dict. <#American_Dictionary>) Ūnĭcōde̽ ?
|
||||
Limbu
|
||||
Linear B
|
||||
Malayalam യൂനികോഡ്
|
||||
Mongolian
|
||||
Myanmar
|
||||
Ogham ᚔᚒᚅᚔᚉᚑᚇ / /
|
||||
Old Italic
|
||||
Oriya ୟୂନିକୋଡ
|
||||
Osmanya
|
||||
Runic (Anglo-Saxon) ᛡᚢᚾᛁᚳᚩᛞ
|
||||
Shavian
|
||||
Sinhala යණනිකෞද්
|
||||
Syriac ܝܘܢܝܩܘܕ
|
||||
Tagbanwa
|
||||
Tagalog
|
||||
Tai Le
|
||||
Tamil யூனிகோட்
|
||||
Telugu యూనికోడ్
|
||||
Thaana
|
||||
Thai ยูนืโคด
|
||||
Tibetan (Dzongkha) ཨུ་ནི་ཀོཌྲ།
|
||||
Ugaritic
|
||||
Yi
|
||||
|
||||
|
||||
Notes:
|
||||
|
||||
There are different ways to transcribe the word “Unicode”, depending on
|
||||
the language and script. In some cases there is only one language that
|
||||
customarily uses a given script; in others there are many languages. The
|
||||
goal here is at a minimum to collect at least one transcription for each
|
||||
script in a language customarily written in that script, with more
|
||||
languages if possible. If the transcription is the same for multiple
|
||||
languages in a script, then a single representative language is used.
|
||||
|
||||
Still missing are transcriptions for the items above in RED (in at least
|
||||
one language). I would appreciate any other transcriptions, or
|
||||
corrections for the ones listed here. Send to mark3@macchiato.com
|
||||
<mailto:mark3@macchiato.com>, using the directions below:
|
||||
|
||||
* *Supplying Missing Items*
|
||||
o Most Latin-script languages will follow the spelling, and
|
||||
change the pronunciation. For any that would not, it would
|
||||
be good to have the alternate spelling.
|
||||
o For non-Latin scripts the goal is to match the English
|
||||
pronunciation — /*not*/ spelling. Above is the IPA <#IPA>
|
||||
(in phonemic transcription) that should be matched as
|
||||
closely as possible (without sounding affected in the target
|
||||
language)
|
||||
o Text would be best in either the UTF-8 text, or the code
|
||||
points in hex HTML. E.g. either of the following:
|
||||
+ "Юникод"
|
||||
+ "Юникод"
|
||||
+ Note: for / supplementary characters/
|
||||
<http://www.unicode.org/glossary/#supplementary_character>,
|
||||
there should be one hex number per code point, not two
|
||||
surrogates
|
||||
<http://www.unicode.org/glossary/#surrogate_code_point>:
|
||||
# 𐀀 /*not*/ �&xDC00;
|
||||
o If you have a good font, I'd also appreciate a GIF. It
|
||||
should be *96 x 24* bits, with the text centered, in black
|
||||
on white (plus grays if smoothed).
|
||||
* *Other Comments*
|
||||
o Because some browsers won't handle the text, both text and
|
||||
GIF image are supplied. If you can’t read the text columns,
|
||||
see Display Problems
|
||||
<http://www.unicode.org/help/display_problems.html>.
|
||||
o The Chinese versions (inc. Bopomofo) are translations, not
|
||||
transcriptions, since "transcription in Chinese is pretty
|
||||
lame" [J. Becker].
|
||||
o There are other "translations" of Unicode that may be in
|
||||
use, such as the Vietnamese "Thống Nhất Mã".
|
||||
o For sample pages in different languages on the Unicode site,
|
||||
see What is Unicode?
|
||||
<http://www.unicode.org/unicode/standard/WhatIsUnicode.html>
|
||||
o Americans are not generally used to IPA, and find a variety
|
||||
of different systems in their dictionaries. This one leaves
|
||||
the base letters as they are, and uses diacritics for
|
||||
pronunciation.
|
||||
* *Etymology of /Unicode/*
|
||||
o Coined by J. Becker. Not related to previous usages, such as:
|
||||
+ A telegraphic code in which one word or set of letters
|
||||
represents a sentence or phrase; a telegram or message
|
||||
in this. (late 19th century, OED)
|
||||
o According to my references, the prefix "uni" is directly
|
||||
from Latin while the word "code" is through French.
|
||||
o The original Indo-European apparently would have been
|
||||
*oino-kau-do ("one strike give"): *kau apparently being
|
||||
related to such English words as: hew, haggle, hoe, hag,
|
||||
hay, hack, caudad, caudal, caudate, caudex, coda, codex,
|
||||
codicil, coward, incus, and Kovač (personal name: "smith").
|
||||
+ I will leave the exact derivations to the exegetes,
|
||||
but I like the association with "haggle" myself.
|
||||
* *Contributions*
|
||||
o This draws on contributions or comments from:
|
||||
+ Dixon Au
|
||||
+ Joe Becker
|
||||
+ Maurice Bauhahn
|
||||
+ Abel Cheung
|
||||
+ Peter Constable
|
||||
+ Michael Everson
|
||||
+ Christopher John Fynn
|
||||
+ Michael Kaplan
|
||||
+ George Kiraz
|
||||
+ Abdul Malik
|
||||
+ Siva Nataraja
|
||||
+ Roozbeh Pournader
|
||||
+ Jonathan Rosenne
|
||||
+ Jungshik Shin
|
||||
|
||||
------------------------------------------------------------------------
|
||||
|
||||
|
||||
Terms of Use <http://www.macchiato.com/terms_of_use.html>. Last updated:
|
||||
MED - 04/20/2003 15:30:33.
|
||||
<http://member.linkexchange.com/cgi-bin/fc/fastcounter-login?750641>
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,126 @@
|
|||
Sentences that contain all letters commonly used in a language
|
||||
--------------------------------------------------------------
|
||||
|
||||
Markus Kuhn <http://www.cl.cam.ac.uk/~mgk25/> -- 2001-09-02
|
||||
|
||||
This file is UTF-8 encoded.
|
||||
|
||||
|
||||
Danish (da)
|
||||
---------
|
||||
|
||||
Quizdeltagerne spiste jordbær med fløde, mens cirkusklovnen
|
||||
Wolther spillede på xylofon.
|
||||
(= Quiz contestants were eating strawbery with cream while Wolther
|
||||
the circus clown played on xylophone.)
|
||||
|
||||
German (de)
|
||||
-----------
|
||||
|
||||
Falsches Üben von Xylophonmusik quält jeden größeren Zwerg
|
||||
(= Wrongful practicing of xylophone music tortures every larger dwarf)
|
||||
|
||||
Zwölf Boxkämpfer jagten Eva quer über den Sylter Deich
|
||||
(= Twelve boxing fighters hunted Eva across the dike of Sylt)
|
||||
|
||||
Heizölrückstoßabdämpfung
|
||||
(= fuel oil recoil absorber)
|
||||
(jqvwxy missing, but all non-ASCII letters in one word)
|
||||
|
||||
English (en)
|
||||
------------
|
||||
|
||||
The quick brown fox jumps over the lazy dog
|
||||
|
||||
Spanish (es)
|
||||
------------
|
||||
|
||||
El pingüino Wenceslao hizo kilómetros bajo exhaustiva lluvia y
|
||||
frío, añoraba a su querido cachorro.
|
||||
(Contains every letter and every accent, but not every combination
|
||||
of vowel + acute.)
|
||||
|
||||
French (fr)
|
||||
-----------
|
||||
|
||||
Portez ce vieux whisky au juge blond qui fume sur son île intérieure, à
|
||||
côté de l'alcôve ovoïde, où les bûches se consument dans l'âtre, ce
|
||||
qui lui permet de penser à la cænogenèse de l'être dont il est question
|
||||
dans la cause ambiguë entendue à Moÿ, dans un capharnaüm qui,
|
||||
pense-t-il, diminue çà et là la qualité de son œuvre.
|
||||
|
||||
l'île exiguë
|
||||
Où l'obèse jury mûr
|
||||
Fête l'haï volapük,
|
||||
Âne ex aéquo au whist,
|
||||
Ôtez ce vœu déçu.
|
||||
|
||||
Le cœur déçu mais l'âme plutôt naïve, Louÿs rêva de crapaüter en
|
||||
canoë au delà des îles, près du mälström où brûlent les novæ.
|
||||
|
||||
Irish Gaelic (ga)
|
||||
-----------------
|
||||
|
||||
D'fhuascail Íosa, Úrmhac na hÓighe Beannaithe, pór Éava agus Ádhaimh
|
||||
|
||||
Hungarian (hu)
|
||||
--------------
|
||||
|
||||
Árvíztűrő tükörfúrógép
|
||||
(= flood-proof mirror-drilling machine, only all non-ASCII letters)
|
||||
|
||||
Icelandic (is)
|
||||
--------------
|
||||
|
||||
Kæmi ný öxi hér ykist þjófum nú bæði víl og ádrepa
|
||||
|
||||
Sævör grét áðan því úlpan var ónýt
|
||||
(some ASCII letters missing)
|
||||
|
||||
Japanese (jp)
|
||||
-------------
|
||||
|
||||
Hiragana: (Iroha)
|
||||
|
||||
いろはにほへとちりぬるを
|
||||
わかよたれそつねならむ
|
||||
うゐのおくやまけふこえて
|
||||
あさきゆめみしゑひもせす
|
||||
|
||||
Katakana:
|
||||
|
||||
イロハニホヘト チリヌルヲ ワカヨタレソ ツネナラム
|
||||
ウヰノオクヤマ ケフコエテ アサキユメミシ ヱヒモセスン
|
||||
|
||||
Hebrew (iw)
|
||||
-----------
|
||||
|
||||
? דג סקרן שט בים מאוכזב ולפתע מצא לו חברה איך הקליטה
|
||||
|
||||
Polish (pl)
|
||||
-----------
|
||||
|
||||
Pchnąć w tę łódź jeża lub ośm skrzyń fig
|
||||
(= To push a hedgehog or eight bins of figs in this boat)
|
||||
|
||||
Russian (ru)
|
||||
------------
|
||||
|
||||
В чащах юга жил бы цитрус? Да, но фальшивый экземпляр!
|
||||
(= Would a citrus live in the bushes of south? Yes, but only a fake one!)
|
||||
|
||||
Thai (th)
|
||||
---------
|
||||
|
||||
[--------------------------|------------------------]
|
||||
๏ เป็นมนุษย์สุดประเสริฐเลิศคุณค่า กว่าบรรดาฝูงสัตว์เดรัจฉาน
|
||||
จงฝ่าฟันพัฒนาวิชาการ อย่าล้างผลาญฤๅเข่นฆ่าบีฑาใคร
|
||||
ไม่ถือโทษโกรธแช่งซัดฮึดฮัดด่า หัดอภัยเหมือนกีฬาอัชฌาสัย
|
||||
ปฏิบัติประพฤติกฎกำหนดใจ พูดจาให้จ๊ะๆ จ๋าๆ น่าฟังเอย ฯ
|
||||
|
||||
[The copyright for the Thai example is owned by The Computer
|
||||
Association of Thailand under the Royal Patronage of His Majesty the
|
||||
King.]
|
||||
|
||||
Please let me know if you find others! Special thanks to the people
|
||||
from all over the world who contributed these sentences.
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
#include "../../source/utf8.h"
|
||||
using namespace utf8;
|
||||
|
||||
#include <string>
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <algorithm>
|
||||
using namespace std;
|
||||
|
||||
const unsigned INVALID_LINES[] = { 75, 76, 83, 84, 85, 93, 102, 103, 105, 106, 107, 108, 109, 110, 114, 115, 116, 117, 124, 125, 130, 135, 140, 145, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 169, 175, 176, 177, 207, 208, 209, 210, 211, 220, 221, 222, 223, 224, 232, 233, 234, 235, 236, 247, 248, 249, 250, 251, 252, 253, 257, 258, 259, 260, 261, 262, 263, 264};
|
||||
const unsigned* INVALID_LINES_END = INVALID_LINES + sizeof(INVALID_LINES)/sizeof(unsigned);
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
string test_file_path;
|
||||
if (argc == 2)
|
||||
test_file_path = argv[1];
|
||||
else {
|
||||
cout << "Wrong number of arguments" << endl;
|
||||
exit(0);
|
||||
}
|
||||
// Open the test file
|
||||
ifstream fs8(test_file_path.c_str());
|
||||
if (!fs8.is_open()) {
|
||||
cout << "Could not open " << test_file_path << endl;
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Read it line by line
|
||||
unsigned int line_count = 0;
|
||||
char byte;
|
||||
while (!fs8.eof()) {
|
||||
string line;
|
||||
while ((byte = static_cast<char>(fs8.get())) != '\n' && !fs8.eof())
|
||||
line.push_back(byte);
|
||||
|
||||
line_count++;
|
||||
bool expected_valid = (find(INVALID_LINES, INVALID_LINES_END, line_count) == INVALID_LINES_END);
|
||||
// Print out lines that contain unexpected invalid UTF-8
|
||||
if (!is_valid(line.begin(), line.end())) {
|
||||
if (expected_valid)
|
||||
cout << "Unexpected invalid utf-8 at line " << line_count << '\n';
|
||||
|
||||
// try fixing it:
|
||||
string fixed_line;
|
||||
replace_invalid(line.begin(), line.end(), back_inserter(fixed_line));
|
||||
if (!is_valid(fixed_line.begin(), fixed_line.end()))
|
||||
cout << "replace_invalid() resulted in an invalid utf-8 at line " << line_count << '\n';
|
||||
}
|
||||
else if (!expected_valid)
|
||||
cout << "Invalid utf-8 NOT detected at line " << line_count << '\n';
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,297 @@
|
|||
#include <cstring>
|
||||
#include <cassert>
|
||||
#include <vector>
|
||||
#include "../../source/utf8.h"
|
||||
using namespace utf8;
|
||||
using namespace std;
|
||||
|
||||
int main()
|
||||
{
|
||||
//append
|
||||
unsigned char u[5] = {0,0,0,0,0};
|
||||
|
||||
append(0x0448, u);
|
||||
assert (u[0] == 0xd1 && u[1] == 0x88 && u[2] == 0 && u[3] == 0 && u[4] == 0);
|
||||
|
||||
append(0x65e5, u);
|
||||
assert (u[0] == 0xe6 && u[1] == 0x97 && u[2] == 0xa5 && u[3] == 0 && u[4] == 0);
|
||||
|
||||
append(0x3044, u);
|
||||
assert (u[0] == 0xe3 && u[1] == 0x81 && u[2] == 0x84 && u[3] == 0 && u[4] == 0);
|
||||
|
||||
append(0x10346, u);
|
||||
assert (u[0] == 0xf0 && u[1] == 0x90 && u[2] == 0x8d && u[3] == 0x86 && u[4] == 0);
|
||||
|
||||
|
||||
//next
|
||||
const char* twochars = "\xe6\x97\xa5\xd1\x88";
|
||||
const char* w = twochars;
|
||||
int cp = next(w, twochars + 6);
|
||||
assert (cp == 0x65e5);
|
||||
assert (w == twochars + 3);
|
||||
|
||||
const char* threechars = "\xf0\x90\x8d\x86\xe6\x97\xa5\xd1\x88";
|
||||
w = threechars;
|
||||
cp = next(w, threechars + 9);
|
||||
assert (cp == 0x10346);
|
||||
assert (w == threechars + 4);
|
||||
cp = next(w, threechars + 9);
|
||||
assert (cp == 0x65e5);
|
||||
assert (w == threechars + 7);
|
||||
cp = next(w, threechars + 9);
|
||||
assert (cp == 0x0448);
|
||||
assert (w == threechars + 9);
|
||||
|
||||
//peek_next
|
||||
const char* const cw = twochars;
|
||||
cp = peek_next(cw, cw + 6);
|
||||
assert (cp == 0x65e5);
|
||||
assert (cw == twochars);
|
||||
|
||||
//prior
|
||||
w = twochars + 3;
|
||||
cp = prior (w, twochars);
|
||||
assert (cp == 0x65e5);
|
||||
assert (w == twochars);
|
||||
|
||||
w = threechars + 9;
|
||||
cp = prior(w, threechars);
|
||||
assert (cp == 0x0448);
|
||||
assert (w == threechars + 7);
|
||||
cp = prior(w, threechars);
|
||||
assert (cp == 0x65e5);
|
||||
assert (w == threechars + 4);
|
||||
cp = prior(w, threechars);
|
||||
assert (cp == 0x10346);
|
||||
assert (w == threechars);
|
||||
|
||||
//previous (deprecated)
|
||||
w = twochars + 3;
|
||||
cp = previous (w, twochars - 1);
|
||||
assert (cp == 0x65e5);
|
||||
assert (w == twochars);
|
||||
|
||||
w = threechars + 9;
|
||||
cp = previous(w, threechars - 1);
|
||||
assert (cp == 0x0448);
|
||||
assert (w == threechars + 7);
|
||||
cp = previous(w, threechars -1);
|
||||
assert (cp == 0x65e5);
|
||||
assert (w == threechars + 4);
|
||||
cp = previous(w, threechars - 1);
|
||||
assert (cp == 0x10346);
|
||||
assert (w == threechars);
|
||||
|
||||
// advance
|
||||
w = twochars;
|
||||
advance (w, 2, twochars + 6);
|
||||
assert (w == twochars + 5);
|
||||
|
||||
// distance
|
||||
size_t dist = utf8::distance(twochars, twochars + 5);
|
||||
assert (dist == 2);
|
||||
|
||||
// utf32to8
|
||||
int utf32string[] = {0x448, 0x65E5, 0x10346, 0};
|
||||
vector<char> utf8result;
|
||||
utf32to8(utf32string, utf32string + 3, back_inserter(utf8result));
|
||||
assert (utf8result.size() == 9);
|
||||
// try it with the return value;
|
||||
char* utf8_end = utf32to8(utf32string, utf32string + 3, &utf8result[0]);
|
||||
assert (utf8_end == &utf8result[0] + 9);
|
||||
|
||||
//utf8to32
|
||||
vector<int> utf32result;
|
||||
utf8to32(twochars, twochars + 5, back_inserter(utf32result));
|
||||
assert (utf32result.size() == 2);
|
||||
// try it with the return value;
|
||||
int* utf32_end = utf8to32(twochars, twochars + 5, &utf32result[0]);
|
||||
assert (utf32_end == &utf32result[0] + 2);
|
||||
|
||||
//utf16to8
|
||||
unsigned short utf16string[] = {0x41, 0x0448, 0x65e5, 0xd834, 0xdd1e};
|
||||
utf8result.clear();
|
||||
utf16to8(utf16string, utf16string + 5, back_inserter(utf8result));
|
||||
assert (utf8result.size() == 10);
|
||||
// try it with the return value;
|
||||
utf8_end = utf16to8 (utf16string, utf16string + 5, &utf8result[0]);
|
||||
assert (utf8_end == &utf8result[0] + 10);
|
||||
|
||||
//utf8to16
|
||||
char utf8_with_surrogates[] = "\xe6\x97\xa5\xd1\x88\xf0\x9d\x84\x9e";
|
||||
vector <unsigned short> utf16result;
|
||||
utf8to16(utf8_with_surrogates, utf8_with_surrogates + 9, back_inserter(utf16result));
|
||||
assert (utf16result.size() == 4);
|
||||
assert (utf16result[2] == 0xd834);
|
||||
assert (utf16result[3] == 0xdd1e);
|
||||
// try it with the return value;
|
||||
unsigned short* utf16_end = utf8to16 (utf8_with_surrogates, utf8_with_surrogates + 9, &utf16result[0]);
|
||||
assert (utf16_end == &utf16result[0] + 4);
|
||||
|
||||
//find_invalid
|
||||
char utf_invalid[] = "\xe6\x97\xa5\xd1\x88\xfa";
|
||||
char* invalid = find_invalid(utf_invalid, utf_invalid + 6);
|
||||
assert (invalid == utf_invalid + 5);
|
||||
|
||||
//is_valid
|
||||
bool bvalid = is_valid(utf_invalid, utf_invalid + 6);
|
||||
assert (bvalid == false);
|
||||
bvalid = is_valid(utf8_with_surrogates, utf8_with_surrogates + 9);
|
||||
assert (bvalid == true);
|
||||
|
||||
//starts_with_bom
|
||||
unsigned char byte_order_mark[] = {0xef, 0xbb, 0xbf};
|
||||
bool bbom = starts_with_bom(byte_order_mark, byte_order_mark + sizeof(byte_order_mark));
|
||||
assert (bbom == true);
|
||||
bool no_bbom = starts_with_bom(threechars, threechars + sizeof(threechars));
|
||||
assert (no_bbom == false);
|
||||
|
||||
//is_bom
|
||||
bool unsafe_bbom = is_bom(byte_order_mark);
|
||||
assert (unsafe_bbom == true);
|
||||
|
||||
|
||||
//replace_invalid
|
||||
char invalid_sequence[] = "a\x80\xe0\xa0\xc0\xaf\xed\xa0\x80z";
|
||||
vector<char> replace_invalid_result(50);
|
||||
replace_invalid (invalid_sequence, invalid_sequence + sizeof(invalid_sequence), replace_invalid_result.begin(), '?');
|
||||
bvalid = is_valid(replace_invalid_result.begin(), replace_invalid_result.end());
|
||||
assert (bvalid);
|
||||
const char* fixed_invalid_sequence = "a????z";
|
||||
assert (std::equal(replace_invalid_result.begin(), replace_invalid_result.begin() + sizeof(fixed_invalid_sequence), fixed_invalid_sequence));
|
||||
|
||||
// iterator
|
||||
utf8::iterator<const char*> it(threechars, threechars, threechars + 9);
|
||||
utf8::iterator<const char*> it2 = it;
|
||||
assert (it2 == it);
|
||||
assert (*it == 0x10346);
|
||||
assert (*(++it) == 0x65e5);
|
||||
assert ((*it++) == 0x65e5);
|
||||
assert (*it == 0x0448);
|
||||
assert (it != it2);
|
||||
utf8::iterator<const char*> endit (threechars + 9, threechars, threechars + 9);
|
||||
assert (++it == endit);
|
||||
assert (*(--it) == 0x0448);
|
||||
assert ((*it--) == 0x0448);
|
||||
assert (*it == 0x65e5);
|
||||
assert (--it == utf8::iterator<const char*>(threechars, threechars, threechars + 9));
|
||||
assert (*it == 0x10346);
|
||||
|
||||
//////////////////////////////////////////////////////////
|
||||
//// Unchecked variants
|
||||
//////////////////////////////////////////////////////////
|
||||
|
||||
//append
|
||||
memset(u, 0, 5);
|
||||
append(0x0448, u);
|
||||
assert (u[0] == 0xd1 && u[1] == 0x88 && u[2] == 0 && u[3] == 0 && u[4] == 0);
|
||||
|
||||
append(0x65e5, u);
|
||||
assert (u[0] == 0xe6 && u[1] == 0x97 && u[2] == 0xa5 && u[3] == 0 && u[4] == 0);
|
||||
|
||||
append(0x10346, u);
|
||||
assert (u[0] == 0xf0 && u[1] == 0x90 && u[2] == 0x8d && u[3] == 0x86 && u[4] == 0);
|
||||
|
||||
//next
|
||||
w = twochars;
|
||||
cp = unchecked::next(w);
|
||||
assert (cp == 0x65e5);
|
||||
assert (w == twochars + 3);
|
||||
|
||||
w = threechars;
|
||||
cp = unchecked::next(w);
|
||||
assert (cp == 0x10346);
|
||||
assert (w == threechars + 4);
|
||||
cp = unchecked::next(w);
|
||||
assert (cp == 0x65e5);
|
||||
assert (w == threechars + 7);
|
||||
cp = unchecked::next(w);
|
||||
assert (cp == 0x0448);
|
||||
assert (w == threechars + 9);
|
||||
|
||||
//peek_next
|
||||
cp = unchecked::peek_next(cw);
|
||||
assert (cp == 0x65e5);
|
||||
assert (cw == twochars);
|
||||
|
||||
|
||||
//previous (calls prior internally)
|
||||
|
||||
w = twochars + 3;
|
||||
cp = unchecked::previous (w);
|
||||
assert (cp == 0x65e5);
|
||||
assert (w == twochars);
|
||||
|
||||
w = threechars + 9;
|
||||
cp = unchecked::previous(w);
|
||||
assert (cp == 0x0448);
|
||||
assert (w == threechars + 7);
|
||||
cp = unchecked::previous(w);
|
||||
assert (cp == 0x65e5);
|
||||
assert (w == threechars + 4);
|
||||
cp = unchecked::previous(w);
|
||||
assert (cp == 0x10346);
|
||||
assert (w == threechars);
|
||||
|
||||
// advance
|
||||
w = twochars;
|
||||
unchecked::advance (w, 2);
|
||||
assert (w == twochars + 5);
|
||||
|
||||
// distance
|
||||
dist = unchecked::distance(twochars, twochars + 5);
|
||||
assert (dist == 2);
|
||||
|
||||
// utf32to8
|
||||
utf8result.clear();
|
||||
unchecked::utf32to8(utf32string, utf32string + 3, back_inserter(utf8result));
|
||||
assert (utf8result.size() == 9);
|
||||
// try it with the return value;
|
||||
utf8_end = utf32to8(utf32string, utf32string + 3, &utf8result[0]);
|
||||
assert(utf8_end == &utf8result[0] + 9);
|
||||
|
||||
//utf8to32
|
||||
utf32result.clear();
|
||||
unchecked::utf8to32(twochars, twochars + 5, back_inserter(utf32result));
|
||||
assert (utf32result.size() == 2);
|
||||
// try it with the return value;
|
||||
utf32_end = utf8to32(twochars, twochars + 5, &utf32result[0]);
|
||||
assert (utf32_end == &utf32result[0] + 2);
|
||||
|
||||
//utf16to8
|
||||
utf8result.clear();
|
||||
unchecked::utf16to8(utf16string, utf16string + 5, back_inserter(utf8result));
|
||||
assert (utf8result.size() == 10);
|
||||
// try it with the return value;
|
||||
utf8_end = utf16to8 (utf16string, utf16string + 5, &utf8result[0]);
|
||||
assert (utf8_end == &utf8result[0] + 10);
|
||||
|
||||
//utf8to16
|
||||
utf16result.clear();
|
||||
unchecked::utf8to16(utf8_with_surrogates, utf8_with_surrogates + 9, back_inserter(utf16result));
|
||||
assert (utf16result.size() == 4);
|
||||
assert (utf16result[2] == 0xd834);
|
||||
assert (utf16result[3] == 0xdd1e);
|
||||
// try it with the return value;
|
||||
utf16_end = utf8to16 (utf8_with_surrogates, utf8_with_surrogates + 9, &utf16result[0]);
|
||||
assert (utf16_end == &utf16result[0] + 4);
|
||||
|
||||
// iterator
|
||||
utf8::unchecked::iterator<const char*> un_it(threechars);
|
||||
utf8::unchecked::iterator<const char*> un_it2 = un_it;
|
||||
assert (un_it2 == un_it);
|
||||
assert (*un_it == 0x10346);
|
||||
assert (*(++un_it) == 0x65e5);
|
||||
assert ((*un_it++) == 0x65e5);
|
||||
assert (un_it != un_it2);
|
||||
assert (*un_it == 0x0448);
|
||||
utf8::unchecked::iterator<const char*> un_endit (threechars + 9);
|
||||
assert (++un_it == un_endit);
|
||||
assert (*(--un_it) == 0x0448);
|
||||
assert ((*un_it--) == 0x0448);
|
||||
assert (*un_it == 0x65e5);
|
||||
assert (--un_it == utf8::unchecked::iterator<const char*>(threechars));
|
||||
assert (*un_it == 0x10346);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,160 @@
|
|||
#include "../../source/utf8.h"
|
||||
using namespace utf8;
|
||||
|
||||
#include <string>
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <vector>
|
||||
using namespace std;
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
if (argc != 2) {
|
||||
cout << "\nUsage: utfreader filename\n";
|
||||
return 0;
|
||||
}
|
||||
const char* TEST_FILE_PATH = argv[1];
|
||||
// Open the test file
|
||||
ifstream fs8(TEST_FILE_PATH);
|
||||
if (!fs8.is_open()) {
|
||||
cout << "Could not open " << TEST_FILE_PATH << endl;
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Read it line by line
|
||||
unsigned int line_count = 0;
|
||||
char byte;
|
||||
while (!fs8.eof()) {
|
||||
string line;
|
||||
while ((byte = static_cast<char>(fs8.get())) != '\n' && !fs8.eof())
|
||||
line.push_back(byte);
|
||||
|
||||
line_count++;
|
||||
// Play around with each line and convert it to utf16
|
||||
string::iterator line_start = line.begin();
|
||||
string::iterator line_end = line.end();
|
||||
line_end = find_invalid(line_start, line_end);
|
||||
if (line_end != line.end())
|
||||
cout << "Line " << line_count << ": Invalid utf-8 at byte " << int(line.end() - line_end) << '\n';
|
||||
|
||||
// Convert it to utf-16 and write to the file
|
||||
vector<unsigned short> utf16_line;
|
||||
utf8to16(line_start, line_end, back_inserter(utf16_line));
|
||||
|
||||
// Back to utf-8 and compare it to the original line.
|
||||
string back_to_utf8;
|
||||
utf16to8(utf16_line.begin(), utf16_line.end(), back_inserter(back_to_utf8));
|
||||
if (back_to_utf8.compare(string(line_start, line_end)) != 0)
|
||||
cout << "Line " << line_count << ": Conversion to UTF-16 and back failed" << '\n';
|
||||
|
||||
// Now, convert it to utf-32, back to utf-8 and compare
|
||||
vector <unsigned> utf32_line;
|
||||
utf8to32(line_start, line_end, back_inserter(utf32_line));
|
||||
back_to_utf8.clear();
|
||||
utf32to8(utf32_line.begin(), utf32_line.end(), back_inserter(back_to_utf8));
|
||||
if (back_to_utf8.compare(string(line_start, line_end)) != 0)
|
||||
cout << "Line " << line_count << ": Conversion to UTF-32 and back failed" << '\n';
|
||||
|
||||
// Now, iterate and back
|
||||
unsigned char_count = 0;
|
||||
string::iterator it = line_start;
|
||||
while (it != line_end) {
|
||||
unsigned int next_cp = peek_next(it, line_end);
|
||||
if (next(it, line_end) != next_cp)
|
||||
cout << "Line " << line_count << ": Error: peek_next gave a different result than next" << '\n';
|
||||
char_count++;
|
||||
}
|
||||
if (char_count != utf32_line.size())
|
||||
cout << "Line " << line_count << ": Error in iterating with next - wrong number of characters" << '\n';
|
||||
|
||||
string::iterator adv_it = line_start;
|
||||
utf8::advance(adv_it, char_count, line_end);
|
||||
if (adv_it != line_end)
|
||||
cout << "Line " << line_count << ": Error in advance function" << '\n';
|
||||
|
||||
if (string::size_type(utf8::distance(line_start, line_end)) != char_count)
|
||||
cout << "Line " << line_count << ": Error in distance function" << '\n';
|
||||
|
||||
while (it != line_start) {
|
||||
previous(it, line.rend().base());
|
||||
char_count--;
|
||||
}
|
||||
if (char_count != 0)
|
||||
cout << "Line " << line_count << ": Error in iterating with previous - wrong number of characters" << '\n';
|
||||
|
||||
// Try utf8::iterator
|
||||
utf8::iterator<string::iterator> u8it(line_start, line_start, line_end);
|
||||
if (!utf32_line.empty() && *u8it != utf32_line.at(0))
|
||||
cout << "Line " << line_count << ": Error in utf::iterator * operator" << '\n';
|
||||
if (std::distance(u8it, utf8::iterator<string::iterator>(line_end, line_start, line_end)) != static_cast<int>(utf32_line.size()))
|
||||
cout << "Line " << line_count << ": Error in using utf::iterator with std::distance - wrong number of characters" << '\n';
|
||||
|
||||
std::advance(u8it, utf32_line.size());
|
||||
if (u8it != utf8::iterator<string::iterator>(line_end, line_start, line_end))
|
||||
cout << "Line " << line_count << ": Error in using utf::iterator with std::advance" << '\n';
|
||||
|
||||
|
||||
//======================== Now, the unchecked versions ======================
|
||||
// Convert it to utf-16 and compare to the checked version
|
||||
vector<unsigned short> utf16_line_unchecked;
|
||||
unchecked::utf8to16(line_start, line_end, back_inserter(utf16_line_unchecked));
|
||||
|
||||
if (utf16_line != utf16_line_unchecked)
|
||||
cout << "Line " << line_count << ": Error in unchecked::utf8to16" << '\n';
|
||||
|
||||
// Back to utf-8 and compare it to the original line.
|
||||
back_to_utf8.clear();
|
||||
unchecked::utf16to8(utf16_line_unchecked.begin(), utf16_line_unchecked.end(), back_inserter(back_to_utf8));
|
||||
if (back_to_utf8.compare(string(line_start, line_end)) != 0)
|
||||
cout << "Line " << line_count << ": Unchecked conversion to UTF-16 and back failed" << '\n';
|
||||
|
||||
// Now, convert it to utf-32, back to utf-8 and compare
|
||||
vector <unsigned> utf32_line_unchecked;
|
||||
unchecked::utf8to32(line_start, line_end, back_inserter(utf32_line_unchecked));
|
||||
if (utf32_line != utf32_line_unchecked)
|
||||
cout << "Line " << line_count << ": Error in unchecked::utf8to32" << '\n';
|
||||
|
||||
back_to_utf8.clear();
|
||||
unchecked::utf32to8(utf32_line.begin(), utf32_line.end(), back_inserter(back_to_utf8));
|
||||
if (back_to_utf8.compare(string(line_start, line_end)) != 0)
|
||||
cout << "Line " << line_count << ": Unchecked conversion to UTF-32 and back failed" << '\n';
|
||||
|
||||
// Now, iterate and back
|
||||
char_count = 0;
|
||||
it = line_start;
|
||||
while (it != line_end) {
|
||||
unsigned int next_cp = unchecked::peek_next(it);
|
||||
if (unchecked::next(it) != next_cp)
|
||||
cout << "Line " << line_count << ": Error: unchecked::peek_next gave a different result than unchecked::next" << '\n';;
|
||||
char_count++;
|
||||
}
|
||||
if (char_count != utf32_line.size())
|
||||
cout << "Line " << line_count << ": Error in iterating with unchecked::next - wrong number of characters" << '\n';
|
||||
|
||||
adv_it = line_start;
|
||||
utf8::unchecked::advance(adv_it, char_count);
|
||||
if (adv_it != line_end)
|
||||
cout << "Line " << line_count << ": Error in unchecked::advance function" << '\n';
|
||||
|
||||
if (string::size_type(utf8::unchecked::distance(line_start, line_end)) != char_count)
|
||||
cout << "Line " << line_count << ": Error in unchecked::distance function" << '\n';
|
||||
|
||||
while (it != line_start) {
|
||||
unchecked::previous(it);
|
||||
char_count--;
|
||||
}
|
||||
if (char_count != 0)
|
||||
cout << "Line " << line_count << ": Error in iterating with unchecked::previous - wrong number of characters" << '\n';
|
||||
|
||||
// Try utf8::unchecked::iterator
|
||||
utf8::unchecked::iterator<string::iterator> un_u8it(line_start);
|
||||
if (!utf32_line.empty() && *un_u8it != utf32_line.at(0))
|
||||
cout << "Line " << line_count << ": Error in utf::unchecked::iterator * operator" << '\n';
|
||||
if (std::distance(un_u8it, utf8::unchecked::iterator<string::iterator>(line_end)) != static_cast<int>(utf32_line.size()))
|
||||
cout << "Line " << line_count << ": Error in using utf::unchecked::iterator with std::distance - wrong number of characters" << '\n';
|
||||
|
||||
std::advance(un_u8it, utf32_line.size());
|
||||
if (un_u8it != utf8::unchecked::iterator<string::iterator>(line_end))
|
||||
cout << "Line " << line_count << ": Error in using utf::unchecked::iterator with std::advance" << '\n';
|
||||
}
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
#pragma once
|
||||
#pragma once
|
||||
|
||||
#include "EnumConverter.h"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,11 +1,17 @@
|
|||
#include "JsonExporter.h"
|
||||
#include "exporterTools.h"
|
||||
#include <utf8.h>
|
||||
|
||||
using std::string;
|
||||
|
||||
string escapeJsonString(const string& s) {
|
||||
// JavaScript uses UTF-16 internally. As a result, character escaping in JSON strings is UTF-16-based.
|
||||
// Convert string to UTF-16
|
||||
std::u16string utf16String;
|
||||
utf8::utf8to16(s.begin(), s.end(), std::back_inserter(utf16String));
|
||||
|
||||
string result;
|
||||
for (char c : s) {
|
||||
for (char16_t c : utf16String) {
|
||||
switch (c) {
|
||||
case '"': result += "\\\""; break;
|
||||
case '\\': result += "\\\\"; break;
|
||||
|
|
@ -15,10 +21,13 @@ string escapeJsonString(const string& s) {
|
|||
case '\r': result += "\\r"; break;
|
||||
case '\t': result += "\\t"; break;
|
||||
default:
|
||||
if (c <= '\x1f') {
|
||||
result += fmt::format("\\u{0:04x}", c);
|
||||
} else {
|
||||
result += c;
|
||||
{
|
||||
bool needsEscaping = c < '\x20' || c >= 0x80;
|
||||
if (needsEscaping) {
|
||||
result += fmt::format("\\u{0:04x}", c);
|
||||
} else {
|
||||
result += static_cast<char>(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,13 +6,13 @@
|
|||
#include "WaveFileReader.h"
|
||||
|
||||
using boost::optional;
|
||||
using std::u32string;
|
||||
using std::string;
|
||||
using boost::filesystem::path;
|
||||
using std::unique_ptr;
|
||||
|
||||
JoiningContinuousTimeline<Shape> animateAudioClip(
|
||||
const AudioClip& audioClip,
|
||||
optional<u32string> dialog,
|
||||
optional<string> dialog,
|
||||
const ShapeSet& targetShapeSet,
|
||||
int maxThreadCount,
|
||||
ProgressSink& progressSink)
|
||||
|
|
@ -32,7 +32,7 @@ unique_ptr<AudioClip> createWaveAudioClip(path filePath) {
|
|||
|
||||
JoiningContinuousTimeline<Shape> animateWaveFile(
|
||||
path filePath,
|
||||
optional<u32string> dialog,
|
||||
optional<string> dialog,
|
||||
const ShapeSet& targetShapeSet,
|
||||
int maxThreadCount,
|
||||
ProgressSink& progressSink)
|
||||
|
|
|
|||
|
|
@ -9,14 +9,14 @@
|
|||
|
||||
JoiningContinuousTimeline<Shape> animateAudioClip(
|
||||
const AudioClip& audioClip,
|
||||
boost::optional<std::u32string> dialog,
|
||||
boost::optional<std::string> dialog,
|
||||
const ShapeSet& targetShapeSet,
|
||||
int maxThreadCount,
|
||||
ProgressSink& progressSink);
|
||||
|
||||
JoiningContinuousTimeline<Shape> animateWaveFile(
|
||||
boost::filesystem::path filePath,
|
||||
boost::optional<std::u32string> dialog,
|
||||
boost::optional<std::string> dialog,
|
||||
const ShapeSet& targetShapeSet,
|
||||
int maxThreadCount,
|
||||
ProgressSink& progressSink);
|
||||
|
|
|
|||
22
src/main.cpp
22
src/main.cpp
|
|
@ -24,10 +24,11 @@
|
|||
#include <boost/iostreams/device/null.hpp>
|
||||
#include "targetShapeSet.h"
|
||||
#include <boost/utility/in_place_factory.hpp>
|
||||
#include "platformTools.h"
|
||||
|
||||
using std::exception;
|
||||
using std::string;
|
||||
using std::u32string;
|
||||
using std::string;
|
||||
using std::vector;
|
||||
using std::unique_ptr;
|
||||
using std::make_unique;
|
||||
|
|
@ -97,7 +98,14 @@ ShapeSet getTargetShapeSet(const string& extendedShapesString) {
|
|||
return result;
|
||||
}
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
int main(int platformArgc, char *platformArgv[]) {
|
||||
// Use UTF-8 throughout
|
||||
useUtf8ForConsole();
|
||||
useUtf8ForBoostFilesystem();
|
||||
|
||||
// Convert command-line arguments to UTF-8
|
||||
const vector<string> args = argsToUtf8(platformArgc, platformArgv);
|
||||
|
||||
auto pausableStderrSink = addPausableStdErrSink(logging::Level::Warn);
|
||||
pausableStderrSink->pause();
|
||||
|
||||
|
|
@ -130,7 +138,11 @@ int main(int argc, char *argv[]) {
|
|||
});
|
||||
|
||||
// Parse command line
|
||||
cmd.parse(argc, argv);
|
||||
{
|
||||
// TCLAP mutates the function argument! Pass a copy.
|
||||
vector<string> argsCopy(args);
|
||||
cmd.parse(argsCopy);
|
||||
}
|
||||
if (quietMode.getValue()) {
|
||||
infoStream = &nullStream;
|
||||
}
|
||||
|
|
@ -146,7 +158,7 @@ int main(int argc, char *argv[]) {
|
|||
}
|
||||
|
||||
logging::infoFormat("Application startup. Command line: {}", join(
|
||||
vector<char*>(argv, argv + argc) | transformed([](char* arg) { return fmt::format("\"{}\"", arg); }), " "));
|
||||
args | transformed([](string arg) { return fmt::format("\"{}\"", arg); }), " "));
|
||||
|
||||
try {
|
||||
*infoStream << fmt::format("Generating lip sync data for {}.", inputFilePath) << std::endl;
|
||||
|
|
@ -158,7 +170,7 @@ int main(int argc, char *argv[]) {
|
|||
// Animate the recording
|
||||
animation = animateWaveFile(
|
||||
inputFilePath,
|
||||
dialogFile.isSet() ? readUtf8File(path(dialogFile.getValue())) : boost::optional<u32string>(),
|
||||
dialogFile.isSet() ? readUtf8File(path(dialogFile.getValue())) : boost::optional<string>(),
|
||||
targetShapeSet,
|
||||
maxThreadCount.getValue(),
|
||||
progressBar);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#include <g2p.h>
|
||||
#include <g2p.h>
|
||||
#include <regex>
|
||||
#include "stringTools.h"
|
||||
#include "logging.h"
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@
|
|||
#include <gsl_util.h>
|
||||
|
||||
using std::string;
|
||||
using std::u32string;
|
||||
using std::vector;
|
||||
using std::regex;
|
||||
using std::map;
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ using std::regex;
|
|||
using std::regex_replace;
|
||||
using std::chrono::duration;
|
||||
using boost::optional;
|
||||
using std::u32string;
|
||||
using std::string;
|
||||
using std::chrono::duration_cast;
|
||||
using std::array;
|
||||
|
||||
|
|
@ -251,7 +251,7 @@ lambda_unique_ptr<ngram_model_t> createDefaultLanguageModel(ps_decoder_t& decode
|
|||
return std::move(result);
|
||||
}
|
||||
|
||||
lambda_unique_ptr<ngram_model_t> createDialogLanguageModel(ps_decoder_t& decoder, const u32string& dialog) {
|
||||
lambda_unique_ptr<ngram_model_t> createDialogLanguageModel(ps_decoder_t& decoder, const string& dialog) {
|
||||
// Split dialog into normalized words
|
||||
vector<string> words = tokenizeText(dialog, [&](const string& word) { return dictionaryContains(*decoder.dict, word); });
|
||||
|
||||
|
|
@ -264,7 +264,7 @@ lambda_unique_ptr<ngram_model_t> createDialogLanguageModel(ps_decoder_t& decoder
|
|||
return createLanguageModel(words, decoder);
|
||||
}
|
||||
|
||||
lambda_unique_ptr<ngram_model_t> createBiasedLanguageModel(ps_decoder_t& decoder, const u32string& dialog) {
|
||||
lambda_unique_ptr<ngram_model_t> createBiasedLanguageModel(ps_decoder_t& decoder, const string& dialog) {
|
||||
auto defaultLanguageModel = createDefaultLanguageModel(decoder);
|
||||
auto dialogLanguageModel = createDialogLanguageModel(decoder, dialog);
|
||||
constexpr int modelCount = 2;
|
||||
|
|
@ -281,7 +281,7 @@ lambda_unique_ptr<ngram_model_t> createBiasedLanguageModel(ps_decoder_t& decoder
|
|||
return std::move(result);
|
||||
}
|
||||
|
||||
lambda_unique_ptr<ps_decoder_t> createDecoder(optional<u32string> dialog) {
|
||||
lambda_unique_ptr<ps_decoder_t> createDecoder(optional<string> dialog) {
|
||||
lambda_unique_ptr<cmd_ln_t> config(
|
||||
cmd_ln_init(
|
||||
nullptr, ps_args(), true,
|
||||
|
|
@ -435,7 +435,7 @@ Timeline<Phone> utteranceToPhones(
|
|||
|
||||
BoundedTimeline<Phone> recognizePhones(
|
||||
const AudioClip& inputAudioClip,
|
||||
optional<u32string> dialog,
|
||||
optional<string> dialog,
|
||||
int maxThreadCount,
|
||||
ProgressSink& progressSink)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -7,6 +7,6 @@
|
|||
|
||||
BoundedTimeline<Phone> recognizePhones(
|
||||
const AudioClip& audioClip,
|
||||
boost::optional<std::u32string> dialog,
|
||||
boost::optional<std::string> dialog,
|
||||
int maxThreadCount,
|
||||
ProgressSink& progressSink);
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ extern "C" {
|
|||
}
|
||||
|
||||
using std::runtime_error;
|
||||
using std::u32string;
|
||||
using std::string;
|
||||
using std::vector;
|
||||
using std::regex;
|
||||
|
|
@ -34,9 +33,12 @@ static const cst_synth_module synth_method_normalize[] = {
|
|||
};
|
||||
|
||||
vector<string> tokenizeViaFlite(const string& text) {
|
||||
// Convert text to ASCII
|
||||
const string asciiText = utf8ToAscii(text);
|
||||
|
||||
// Create utterance object with text
|
||||
lambda_unique_ptr<cst_utterance> utterance(new_utterance(), [](cst_utterance* utterance) { delete_utterance(utterance); });
|
||||
utt_set_input_text(utterance.get(), text.c_str());
|
||||
utt_set_input_text(utterance.get(), asciiText.c_str());
|
||||
lambda_unique_ptr<cst_voice> voice = createDummyVoice();
|
||||
utt_init(utterance.get(), voice.get());
|
||||
|
||||
|
|
@ -73,8 +75,8 @@ optional<string> findSimilarDictionaryWord(const string& word, function<bool(con
|
|||
return boost::none;
|
||||
}
|
||||
|
||||
vector<string> tokenizeText(const u32string& text, function<bool(const string&)> dictionaryContains) {
|
||||
vector<string> words = tokenizeViaFlite(toAscii(text));
|
||||
vector<string> tokenizeText(const string& text, function<bool(const string&)> dictionaryContains) {
|
||||
vector<string> words = tokenizeViaFlite(text);
|
||||
|
||||
// Join words separated by apostophes
|
||||
for (int i = words.size() - 1; i > 0; --i) {
|
||||
|
|
|
|||
|
|
@ -4,4 +4,4 @@
|
|||
#include <functional>
|
||||
#include <string>
|
||||
|
||||
std::vector<std::string> tokenizeText(const std::u32string& text, std::function<bool(const std::string&)> dictionaryContains);
|
||||
std::vector<std::string> tokenizeText(const std::string& text, std::function<bool(const std::string&)> dictionaryContains);
|
||||
|
|
|
|||
|
|
@ -1,94 +0,0 @@
|
|||
// Generated by asciiCases.rb; don't modify by hand!
|
||||
|
||||
case U'À': case U'Á': case U'Â': case U'Ã': case U'Ä': case U'Å': case U'Ā': case U'Ă': case U'Ą': case U'Ǎ': case U'Ǟ': case U'Ǡ': case U'Ǻ': case U'Ȁ': case U'Ȃ': case U'Ȧ': case U'Ⱥ':
|
||||
return 'A';
|
||||
case U'Ɓ': case U'Ƃ': case U'Ƀ':
|
||||
return 'B';
|
||||
case U'Ç': case U'Ć': case U'Ĉ': case U'Ċ': case U'Č': case U'Ƈ': case U'Ȼ':
|
||||
return 'C';
|
||||
case U'Ď': case U'Đ': case U'Ɗ': case U'Ƌ':
|
||||
return 'D';
|
||||
case U'È': case U'É': case U'Ê': case U'Ë': case U'Ē': case U'Ĕ': case U'Ė': case U'Ę': case U'Ě': case U'Ȅ': case U'Ȇ': case U'Ȩ': case U'Ɇ':
|
||||
return 'E';
|
||||
case U'Ƒ':
|
||||
return 'F';
|
||||
case U'Ĝ': case U'Ğ': case U'Ġ': case U'Ģ': case U'Ɠ': case U'Ǥ': case U'Ǧ': case U'Ǵ':
|
||||
return 'G';
|
||||
case U'Ĥ': case U'Ħ': case U'Ȟ':
|
||||
return 'H';
|
||||
case U'Ì': case U'Í': case U'Î': case U'Ï': case U'Ĩ': case U'Ī': case U'Ĭ': case U'Į': case U'İ': case U'Ɨ': case U'Ǐ': case U'Ȉ': case U'Ȋ':
|
||||
return 'I';
|
||||
case U'Ĵ': case U'Ɉ':
|
||||
return 'J';
|
||||
case U'Ķ': case U'Ƙ': case U'Ǩ':
|
||||
return 'K';
|
||||
case U'Ĺ': case U'Ļ': case U'Ľ': case U'Ŀ': case U'Ł': case U'Ƚ':
|
||||
return 'L';
|
||||
case U'Ñ': case U'Ń': case U'Ņ': case U'Ň': case U'Ɲ': case U'Ǹ': case U'Ƞ':
|
||||
return 'N';
|
||||
case U'Ò': case U'Ó': case U'Ô': case U'Õ': case U'Ö': case U'Ø': case U'Ō': case U'Ŏ': case U'Ő': case U'Ɵ': case U'Ơ': case U'Ǒ': case U'Ǫ': case U'Ǭ': case U'Ǿ': case U'Ȍ': case U'Ȏ': case U'Ȫ': case U'Ȭ': case U'Ȯ': case U'Ȱ':
|
||||
return 'O';
|
||||
case U'Ƥ':
|
||||
return 'P';
|
||||
case U'Ŕ': case U'Ŗ': case U'Ř': case U'Ȑ': case U'Ȓ': case U'Ɍ':
|
||||
return 'R';
|
||||
case U'Ś': case U'Ŝ': case U'Ş': case U'Š': case U'Ș':
|
||||
return 'S';
|
||||
case U'Ţ': case U'Ť': case U'Ŧ': case U'Ƭ': case U'Ʈ': case U'Ț': case U'Ⱦ':
|
||||
return 'T';
|
||||
case U'Ù': case U'Ú': case U'Û': case U'Ü': case U'Ũ': case U'Ū': case U'Ŭ': case U'Ů': case U'Ű': case U'Ų': case U'Ư': case U'Ǔ': case U'Ǖ': case U'Ǘ': case U'Ǚ': case U'Ǜ': case U'Ȕ': case U'Ȗ': case U'Ʉ':
|
||||
return 'U';
|
||||
case U'Ʋ':
|
||||
return 'V';
|
||||
case U'Ŵ':
|
||||
return 'W';
|
||||
case U'Ý': case U'Ŷ': case U'Ÿ': case U'Ƴ': case U'Ȳ': case U'Ɏ':
|
||||
return 'Y';
|
||||
case U'Ź': case U'Ż': case U'Ž': case U'Ƶ': case U'Ȥ':
|
||||
return 'Z';
|
||||
case U'à': case U'á': case U'â': case U'ã': case U'ä': case U'å': case U'ā': case U'ă': case U'ą': case U'ǎ': case U'ǟ': case U'ǡ': case U'ǻ': case U'ȁ': case U'ȃ': case U'ȧ':
|
||||
return 'a';
|
||||
case U'ƀ': case U'ƃ':
|
||||
return 'b';
|
||||
case U'ç': case U'ć': case U'ĉ': case U'ċ': case U'č': case U'ƈ': case U'ȼ':
|
||||
return 'c';
|
||||
case U'ď': case U'đ': case U'ƌ': case U'ȡ':
|
||||
return 'd';
|
||||
case U'è': case U'é': case U'ê': case U'ë': case U'ē': case U'ĕ': case U'ė': case U'ę': case U'ě': case U'ȅ': case U'ȇ': case U'ȩ': case U'ɇ':
|
||||
return 'e';
|
||||
case U'ƒ':
|
||||
return 'f';
|
||||
case U'ĝ': case U'ğ': case U'ġ': case U'ģ': case U'ǥ': case U'ǧ': case U'ǵ':
|
||||
return 'g';
|
||||
case U'ĥ': case U'ħ': case U'ȟ':
|
||||
return 'h';
|
||||
case U'ì': case U'í': case U'î': case U'ï': case U'ĩ': case U'ī': case U'ĭ': case U'į': case U'ǐ': case U'ȉ': case U'ȋ':
|
||||
return 'i';
|
||||
case U'ĵ': case U'ǰ': case U'ɉ':
|
||||
return 'j';
|
||||
case U'ķ': case U'ƙ': case U'ǩ':
|
||||
return 'k';
|
||||
case U'ĺ': case U'ļ': case U'ľ': case U'ŀ': case U'ł': case U'ƚ': case U'ȴ':
|
||||
return 'l';
|
||||
case U'ñ': case U'ń': case U'ņ': case U'ň': case U'ʼn': case U'ƞ': case U'ǹ': case U'ȵ':
|
||||
return 'n';
|
||||
case U'ò': case U'ó': case U'ô': case U'õ': case U'ö': case U'ø': case U'ō': case U'ŏ': case U'ő': case U'ơ': case U'ǒ': case U'ǫ': case U'ǭ': case U'ǿ': case U'ȍ': case U'ȏ': case U'ȫ': case U'ȭ': case U'ȯ': case U'ȱ':
|
||||
return 'o';
|
||||
case U'ƥ':
|
||||
return 'p';
|
||||
case U'ɋ':
|
||||
return 'q';
|
||||
case U'ŕ': case U'ŗ': case U'ř': case U'ȑ': case U'ȓ': case U'ɍ':
|
||||
return 'r';
|
||||
case U'ś': case U'ŝ': case U'ş': case U'š': case U'ș': case U'ȿ':
|
||||
return 's';
|
||||
case U'ţ': case U'ť': case U'ŧ': case U'ƫ': case U'ƭ': case U'ț': case U'ȶ':
|
||||
return 't';
|
||||
case U'ù': case U'ú': case U'û': case U'ü': case U'ũ': case U'ū': case U'ŭ': case U'ů': case U'ű': case U'ų': case U'ư': case U'ǔ': case U'ǖ': case U'ǘ': case U'ǚ': case U'ǜ': case U'ȕ': case U'ȗ':
|
||||
return 'u';
|
||||
case U'ŵ':
|
||||
return 'w';
|
||||
case U'ý': case U'ÿ': case U'ŷ': case U'ƴ': case U'ȳ': case U'ɏ':
|
||||
return 'y';
|
||||
case U'ź': case U'ż': case U'ž': case U'ƶ': case U'ȥ': case U'ɀ':
|
||||
return 'z';
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
require 'open-uri'
|
||||
require 'csv'
|
||||
|
||||
# Create mapping from ASCII characters to related Unicode characters
|
||||
mapping = Hash.new{ |hash, key| hash[key] = [] }
|
||||
url = 'http://www.unicode.org/Public/8.0.0/ucd/UnicodeData.txt'
|
||||
headers = [:code, :name, :category]
|
||||
CSV.new(open(url), :col_sep => ';', :headers => headers).each do |row|
|
||||
code = row[:code].hex
|
||||
next if code < 0x80
|
||||
break if code > 0x24f
|
||||
|
||||
char = [code].pack('U')
|
||||
name = row[:name]
|
||||
match = /^LATIN (CAPITAL|SMALL) LETTER ([A-Z])\b(?!.*\bLETTER\b)/.match(name)
|
||||
if match
|
||||
baseChar = match[2]
|
||||
if match[1] == 'SMALL'
|
||||
baseChar = (baseChar.ord + 0x20).chr
|
||||
end
|
||||
mapping[baseChar] << char
|
||||
end
|
||||
end
|
||||
mapping = mapping.sort.to_h
|
||||
|
||||
# Generate asciiCases.cpp
|
||||
File.open('asciiCases.cpp', 'w') do |file|
|
||||
file.print "// Generated by #{__FILE__}; don't modify by hand!\n\n"
|
||||
mapping.each do |asciiChar, unicodeChars|
|
||||
file.print unicodeChars.map { |c| "case U'#{c}':" }.join(' '), "\n"
|
||||
file.print "\treturn '#{asciiChar}';\n"
|
||||
end
|
||||
end
|
||||
|
|
@ -6,9 +6,21 @@
|
|||
#include <boost/uuid/uuid_io.hpp>
|
||||
#include "platformTools.h"
|
||||
#include <whereami.h>
|
||||
#include <utf8.h>
|
||||
#include <gsl_util.h>
|
||||
#include "tools.h"
|
||||
#include <boost/filesystem/detail/utf8_codecvt_facet.hpp>
|
||||
#include <iostream>
|
||||
|
||||
#ifdef _WIN32
|
||||
#include <Windows.h>
|
||||
#include <io.h>
|
||||
#include <fcntl.h>
|
||||
#endif
|
||||
|
||||
using boost::filesystem::path;
|
||||
using std::string;
|
||||
using std::vector;
|
||||
|
||||
path getBinPath() {
|
||||
static const path binPath = [] {
|
||||
|
|
@ -69,3 +81,73 @@ std::string errorNumberToString(int errorNumber) {
|
|||
#endif
|
||||
return message;
|
||||
}
|
||||
|
||||
vector<string> argsToUtf8(int argc, char* argv[]) {
|
||||
#ifdef _WIN32
|
||||
// On Windows, there is no way to convert the single-byte argument strings to Unicode.
|
||||
// We'll just ignore them.
|
||||
UNUSED(argc);
|
||||
UNUSED(argv);
|
||||
|
||||
// Get command-line arguments as UTF16 strings
|
||||
int argumentCount;
|
||||
static_assert(sizeof(wchar_t) == sizeof(char16_t), "Expected wchar_t to be a 16-bit type.");
|
||||
char16_t** args = reinterpret_cast<char16_t**>(CommandLineToArgvW(GetCommandLineW(), &argumentCount));
|
||||
if (!args) {
|
||||
throw std::runtime_error("Error splitting the UTF-16 command line arguments.");
|
||||
}
|
||||
auto freeArgs = gsl::finally([&]() { LocalFree(args); });
|
||||
assert(argumentCount == argc);
|
||||
|
||||
// Convert UTF16 strings to UTF8
|
||||
vector<string> result;
|
||||
for (int i = 0; i < argc; ++i) {
|
||||
std::u16string utf16String(args[i]);
|
||||
string utf8String;
|
||||
utf8::utf16to8(utf16String.begin(), utf16String.end(), back_inserter(utf8String));
|
||||
result.push_back(utf8String);
|
||||
}
|
||||
return result;
|
||||
#else
|
||||
// On Unix systems, command-line args are already in UTF-8 format. Just convert them to strings.
|
||||
vector<string> result;
|
||||
for (int i = 0; i < argc; ++i) {
|
||||
result.push_back(string(argv[i]));
|
||||
}
|
||||
return result;
|
||||
#endif
|
||||
}
|
||||
|
||||
class ConsoleBuffer : public std::stringbuf {
|
||||
public:
|
||||
explicit ConsoleBuffer(FILE* file)
|
||||
: file(file) {}
|
||||
|
||||
int sync() override {
|
||||
fputs(str().c_str(), file);
|
||||
str("");
|
||||
return 0;
|
||||
}
|
||||
|
||||
private:
|
||||
FILE* file;
|
||||
};
|
||||
|
||||
void useUtf8ForConsole() {
|
||||
// Unix systems already expect UTF-8-encoded data
|
||||
#ifdef _WIN32
|
||||
// Set console code page to UTF-8 so the console knows how to interpret string data
|
||||
SetConsoleOutputCP(CP_UTF8);
|
||||
|
||||
// Prevent default stream buffer from chopping up UTF-8 byte sequences.
|
||||
// See https://stackoverflow.com/questions/45575863/how-to-print-utf-8-strings-to-stdcout-on-windows
|
||||
std::cout.rdbuf(new ConsoleBuffer(stdout));
|
||||
std::cerr.rdbuf(new ConsoleBuffer(stderr));
|
||||
#endif
|
||||
}
|
||||
|
||||
void useUtf8ForBoostFilesystem() {
|
||||
std::locale globalLocale = std::locale();
|
||||
std::locale utf8Locale(globalLocale, new boost::filesystem::detail::utf8_codecvt_facet);
|
||||
path::imbue(utf8Locale);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,3 +10,8 @@ boost::filesystem::path getTempFilePath();
|
|||
|
||||
std::tm getLocalTime(const time_t& time);
|
||||
std::string errorNumberToString(int errorNumber);
|
||||
|
||||
std::vector<std::string> argsToUtf8(int argc, char *argv[]);
|
||||
|
||||
void useUtf8ForConsole();
|
||||
void useUtf8ForBoostFilesystem();
|
||||
|
|
@ -1,12 +1,16 @@
|
|||
#include "stringTools.h"
|
||||
#include "stringTools.h"
|
||||
#include <boost/algorithm/string/trim.hpp>
|
||||
#include <codecvt>
|
||||
#include <utf8.h>
|
||||
#include <utf8proc.h>
|
||||
#include <regex>
|
||||
|
||||
using std::string;
|
||||
using std::wstring;
|
||||
using std::u32string;
|
||||
using std::vector;
|
||||
using boost::optional;
|
||||
using std::regex;
|
||||
using std::regex_replace;
|
||||
|
||||
vector<string> splitIntoLines(const string& s) {
|
||||
vector<string> lines;
|
||||
|
|
@ -83,6 +87,10 @@ vector<string> wrapString(const string& s, int lineLength, int hangingIndent) {
|
|||
return lines;
|
||||
}
|
||||
|
||||
bool isValidUtf8(const string& s) {
|
||||
return utf8::is_valid(s.begin(), s.end());
|
||||
}
|
||||
|
||||
wstring latin1ToWide(const string& s) {
|
||||
wstring result;
|
||||
for (unsigned char c : s) {
|
||||
|
|
@ -91,40 +99,61 @@ wstring latin1ToWide(const string& s) {
|
|||
return result;
|
||||
}
|
||||
|
||||
optional<char> toAscii(char32_t ch) {
|
||||
switch (ch) {
|
||||
#include "asciiCases.cpp"
|
||||
default:
|
||||
return ch < 0x80 ? static_cast<char>(ch) : optional<char>();
|
||||
}
|
||||
}
|
||||
string utf8ToAscii(const string s) {
|
||||
// Normalize string, simplifying it as much as possible
|
||||
const NormalizationOptions options = NormalizationOptions::CompatibilityMode
|
||||
| NormalizationOptions::Decompose
|
||||
| NormalizationOptions::SimplifyLineBreaks
|
||||
| NormalizationOptions::SimplifyWhiteSpace
|
||||
| NormalizationOptions::StripCharacterMarkings
|
||||
| NormalizationOptions::StripIgnorableCharacters;
|
||||
string simplified = normalizeUnicode(s, options);
|
||||
|
||||
string toAscii(const u32string& s) {
|
||||
string result;
|
||||
for (char32_t ch : s) {
|
||||
optional<char> ascii = toAscii(ch);
|
||||
if (ascii) result.append(1, *ascii);
|
||||
// Replace common Unicode characters with ASCII equivalents
|
||||
static const vector<std::pair<regex, string>> replacements{
|
||||
{regex("«|»|“|”|„|‟"), "\""},
|
||||
{regex("‘|’|‚|‛|‹|›"), "'"},
|
||||
{regex("‐|‑|‒|⁃|⁻|₋|−|➖|–|—|―|﹘|﹣|－"), "-"},
|
||||
{regex("…|⋯"), "..."},
|
||||
{regex("•"), "*"},
|
||||
{regex("†|＋"), "+"},
|
||||
{regex("⁄|∕|⧸|／|/"), "/"},
|
||||
{regex("×"), "x"},
|
||||
};
|
||||
for (const auto& replacement : replacements) {
|
||||
simplified = regex_replace(simplified, replacement.first, replacement.second);
|
||||
}
|
||||
|
||||
// Skip all non-ASCII code points, including multi-byte characters
|
||||
string result;
|
||||
for (char c : simplified) {
|
||||
const bool isAscii = (c & 0x80) == 0;
|
||||
if (isAscii) {
|
||||
result.append(1, c);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
string toAscii(const wstring& s) {
|
||||
string result;
|
||||
for (wchar_t ch : s) {
|
||||
optional<char> ascii = toAscii(ch);
|
||||
if (ascii) result.append(1, *ascii);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
string normalizeUnicode(const string s, NormalizationOptions options) {
|
||||
char* result;
|
||||
const utf8proc_ssize_t charCount = utf8proc_map(
|
||||
reinterpret_cast<const uint8_t*>(s.data()),
|
||||
s.length(),
|
||||
reinterpret_cast<uint8_t**>(&result),
|
||||
static_cast<utf8proc_option_t>(options));
|
||||
|
||||
u32string utf8ToUtf32(const string& s) {
|
||||
#if defined(_MSC_VER) && _MSC_VER <= 1900
|
||||
// Workaround for Visual Studio 2015
|
||||
// See https://connect.microsoft.com/VisualStudio/feedback/details/1403302/unresolved-external-when-using-codecvt-utf8
|
||||
std::wstring_convert<std::codecvt_utf8<uint32_t>, uint32_t> convert;
|
||||
return u32string(reinterpret_cast<const char32_t*>(convert.from_bytes(s).c_str()));
|
||||
#else
|
||||
std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> convert;
|
||||
return convert.from_bytes(s);
|
||||
#endif
|
||||
if (charCount < 0) {
|
||||
const utf8proc_ssize_t errorCode = charCount;
|
||||
const string message = string("Error normalizing string: ") + utf8proc_errmsg(errorCode);
|
||||
if (errorCode == UTF8PROC_ERROR_INVALIDOPTS) {
|
||||
throw std::invalid_argument(message);
|
||||
}
|
||||
throw std::runtime_error(message);
|
||||
}
|
||||
|
||||
string resultString(result, charCount);
|
||||
free(result);
|
||||
return resultString;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
#include <vector>
|
||||
#include <boost/optional.hpp>
|
||||
#include <boost/lexical_cast.hpp>
|
||||
#include <utf8proc.h>
|
||||
|
||||
std::vector<std::string> splitIntoLines(const std::string& s);
|
||||
|
||||
|
|
@ -10,15 +11,31 @@ std::vector<std::string> wrapSingleLineString(const std::string& s, int lineLeng
|
|||
|
||||
std::vector<std::string> wrapString(const std::string& s, int lineLength, int hangingIndent = 0);
|
||||
|
||||
bool isValidUtf8(const std::string& s);
|
||||
|
||||
std::wstring latin1ToWide(const std::string& s);
|
||||
|
||||
boost::optional<char> toAscii(char32_t ch);
|
||||
|
||||
std::string toAscii(const std::u32string& s);
|
||||
std::string utf8ToAscii(const std::string s);
|
||||
|
||||
std::string toAscii(const std::wstring& s);
|
||||
enum class NormalizationOptions : int {
|
||||
CompatibilityMode = UTF8PROC_COMPAT,
|
||||
Compose = UTF8PROC_COMPOSE,
|
||||
Decompose = UTF8PROC_DECOMPOSE,
|
||||
StripIgnorableCharacters = UTF8PROC_IGNORE,
|
||||
ThrowOnUnassignedCodepoints = UTF8PROC_REJECTNA,
|
||||
SimplifyLineBreaks = UTF8PROC_NLF2LS,
|
||||
SimplifyWhiteSpace = UTF8PROC_STRIPCC,
|
||||
StripCharacterMarkings = UTF8PROC_STRIPMARK
|
||||
};
|
||||
|
||||
std::u32string utf8ToUtf32(const std::string& s);
|
||||
constexpr NormalizationOptions
|
||||
operator|(NormalizationOptions a, NormalizationOptions b) {
|
||||
return static_cast<NormalizationOptions>(static_cast<int>(a) | static_cast<int>(b));
|
||||
}
|
||||
|
||||
std::string normalizeUnicode(const std::string s, NormalizationOptions options);
|
||||
|
||||
template<typename T>
|
||||
std::string join(T range, const std::string separator) {
|
||||
|
|
|
|||
|
|
@ -5,10 +5,9 @@
|
|||
#include "stringTools.h"
|
||||
|
||||
using std::string;
|
||||
using std::u32string;
|
||||
using boost::filesystem::path;
|
||||
|
||||
u32string readUtf8File(path filePath) {
|
||||
string readUtf8File(path filePath) {
|
||||
if (!exists(filePath)) {
|
||||
throw std::invalid_argument(fmt::format("File {} does not exist.", filePath));
|
||||
}
|
||||
|
|
@ -16,12 +15,12 @@ u32string readUtf8File(path filePath) {
|
|||
boost::filesystem::ifstream file;
|
||||
file.exceptions(std::ifstream::failbit | std::ifstream::badbit);
|
||||
file.open(filePath);
|
||||
string utf8Text((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
|
||||
try {
|
||||
return utf8ToUtf32(utf8Text);
|
||||
} catch (...) {
|
||||
std::throw_with_nested(std::runtime_error(fmt::format("File encoding is not ASCII or UTF-8.", filePath)));
|
||||
string text((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
|
||||
if (!isValidUtf8(text)) {
|
||||
throw std::runtime_error("File encoding is not ASCII or UTF-8.");
|
||||
}
|
||||
|
||||
return text;
|
||||
} catch (...) {
|
||||
std::throw_with_nested(std::runtime_error(fmt::format("Error reading file {0}.", filePath)));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,4 +2,4 @@
|
|||
|
||||
#include <boost/filesystem/path.hpp>
|
||||
|
||||
std::u32string readUtf8File(boost::filesystem::path filePath);
|
||||
std::string readUtf8File(boost::filesystem::path filePath);
|
||||
|
|
@ -81,11 +81,15 @@ TEST(latin1ToWide, basic) {
|
|||
EXPECT_EQ(pangramWide, latin1ToWide(pangramLatin1));
|
||||
}
|
||||
|
||||
// toAscii
|
||||
// utf8ToAscii
|
||||
|
||||
TEST(toAscii, string) {
|
||||
TEST(utf8ToAscii, string) {
|
||||
EXPECT_EQ(
|
||||
"A naive man called was having pina colada and creme brulee.",
|
||||
toAscii(U"A naïve man called 晨 was having piña colada and crème brûlée."));
|
||||
EXPECT_EQ(string(""), toAscii(U""));
|
||||
utf8ToAscii("A naïve man called 晨 was having piña colada and crème brûlée."));
|
||||
EXPECT_EQ(string(""), utf8ToAscii(""));
|
||||
EXPECT_EQ(string("- - - - - - - - - -"), utf8ToAscii("- ‐ ‑ ‒ – — ― ﹘ ﹣ －"));
|
||||
EXPECT_EQ(string("' ' ' ' \" \" \" \" \" \""), utf8ToAscii("‘ ’ ‚ ‛ “ ” „ ‟ « »"));
|
||||
EXPECT_EQ(string("1 2 3"), utf8ToAscii("¹ ² ³"));
|
||||
EXPECT_EQ(string("1/4 1/2 3/4"), utf8ToAscii("¼ ½ ¾"));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
#include "tokenization.h"
|
||||
#include <regex>
|
||||
#include <unordered_set>
|
||||
#include <utf8.h>
|
||||
|
||||
using namespace testing;
|
||||
using std::string;
|
||||
|
|
@ -14,57 +15,64 @@ bool returnTrue(const string&) {
|
|||
}
|
||||
|
||||
TEST(tokenizeText, simpleCases) {
|
||||
EXPECT_THAT(tokenizeText(U"", returnTrue), IsEmpty());
|
||||
EXPECT_THAT(tokenizeText(U" \t\n\r\n ", returnTrue), IsEmpty());
|
||||
EXPECT_THAT(tokenizeText("", returnTrue), IsEmpty());
|
||||
EXPECT_THAT(tokenizeText(" \t\n\r\n ", returnTrue), IsEmpty());
|
||||
EXPECT_THAT(
|
||||
tokenizeText(U"Wit is educated insolence.", returnTrue),
|
||||
tokenizeText("Wit is educated insolence.", returnTrue),
|
||||
ElementsAre("wit", "is", "educated", "insolence")
|
||||
);
|
||||
}
|
||||
|
||||
TEST(tokenizeText, numbers) {
|
||||
EXPECT_THAT(
|
||||
tokenizeText(U"Henry V died at 36.", returnTrue),
|
||||
tokenizeText("Henry V died at 36.", returnTrue),
|
||||
ElementsAre("henry", "the", "fifth", "died", "at", "thirty", "six")
|
||||
);
|
||||
EXPECT_THAT(
|
||||
tokenizeText(U"I spent $4.50 on gum.", returnTrue),
|
||||
tokenizeText("I spent $4.50 on gum.", returnTrue),
|
||||
ElementsAre("i", "spent", "four", "dollars", "fifty", "cents", "on", "gum")
|
||||
);
|
||||
EXPECT_THAT(
|
||||
tokenizeText(U"I was born in 1982.", returnTrue),
|
||||
tokenizeText("I was born in 1982.", returnTrue),
|
||||
ElementsAre("i", "was", "born", "in", "nineteen", "eighty", "two")
|
||||
);
|
||||
}
|
||||
|
||||
TEST(tokenizeText, abbreviations) {
|
||||
EXPECT_THAT(
|
||||
tokenizeText(U"Prof. Foo lives on Dr. Dolittle Dr.", [](const string& word) { return word == "prof."; }),
|
||||
tokenizeText("Prof. Foo lives on Dr. Dolittle Dr.", [](const string& word) { return word == "prof."; }),
|
||||
ElementsAre("prof.", "foo", "lives", "on", "doctor", "dolittle", "drive")
|
||||
);
|
||||
}
|
||||
|
||||
TEST(tokenizeText, apostrophes) {
|
||||
EXPECT_THAT(
|
||||
tokenizeText(U"'Tis said he'd wish'd for a 'bus 'cause he wouldn't walk.", [](const string& word) { return word == "wouldn't"; }),
|
||||
tokenizeText("'Tis said he'd wish'd for a 'bus 'cause he wouldn't walk.", [](const string& word) { return word == "wouldn't"; }),
|
||||
ElementsAreArray(vector<string>{ "tis", "said", "he'd", "wish'd", "for", "a", "bus", "cause", "he", "wouldn't", "walk" })
|
||||
);
|
||||
}
|
||||
|
||||
TEST(tokenizeText, math) {
|
||||
EXPECT_THAT(
|
||||
tokenizeText(U"'1+2*3=7", returnTrue),
|
||||
tokenizeText("'1+2*3=7", returnTrue),
|
||||
ElementsAre("one", "plus", "two", "times", "three", "equals", "seven")
|
||||
);
|
||||
}
|
||||
|
||||
TEST(tokenizeText, unicodeCharacters) {
|
||||
EXPECT_THAT(
|
||||
tokenizeText("A naïve man called 晨 had piña colada and crème brûlée.", returnTrue),
|
||||
ElementsAre("a", "naive", "man", "called", "had", "pina", "colada", "and", "creme", "brulee")
|
||||
);
|
||||
}
|
||||
|
||||
// Checks that each word contains only the characters a-z and the apostrophe
|
||||
TEST(tokenizeText, wordsUseLimitedCharacters) {
|
||||
// Create string containing lots of undesirable characters
|
||||
u32string input = U"A naïve man called 晨 was having piña colada and crème brûlée.";
|
||||
string input = "A naïve man called 晨 was having piña colada and crème brûlée.";
|
||||
for (char32_t c = 0; c <= 1000; ++c) {
|
||||
input.append(U" ");
|
||||
input.append(1, c);
|
||||
input.append(" ");
|
||||
utf8::append(c, back_inserter(input));
|
||||
}
|
||||
|
||||
regex legal("^[a-z']+$");
|
||||
|
|
|
|||
Loading…
Reference in New Issue