utf8proc -> libmojibake (closes JuliaLang#7656, fixes JuliaLang#7582)

.gitmodules

@@ -14,3 +14,6 @@
 	path = doc/juliadoc
 	url = git://github.com/JuliaLang/JuliaDoc.git
 
+[submodule "deps/libmojibake"]
+	path = deps/libmojibake
+	url = git://github.com/JuliaLang/libmojibake.git

Make.inc

@@ -29,7 +29,7 @@ USE_SYSTEM_ZLIB=0
 USE_SYSTEM_GRISU=0
 USE_SYSTEM_RMATH=0
 USE_SYSTEM_LIBUV=0
-USE_SYSTEM_UTF8PROC=0
+USE_SYSTEM_MOJIBAKE=0
 
 # Link to the LLVM shared library
 USE_LLVM_SHLIB = 0
@@ -531,10 +531,10 @@ else
   LIBUV_INC = $(JULIAHOME)/deps/libuv/include
 endif
 
-ifeq ($(USE_SYSTEM_UTF8PROC), 1)
-  LIBUTF8PROC = -lutf8proc
+ifeq ($(USE_SYSTEM_MOJIBAKE), 1)
+  LIBMOJIBAKE = -lmojibake
 else
-  LIBUTF8PROC = $(build_libdir)/libutf8proc.a
+  LIBMOJIBAKE = $(build_libdir)/libmojibake.a
 endif
 
 # OS specific stuff

deps/Makefile

@@ -19,7 +19,7 @@ CONFIGURE_COMMON += F77="$(FC)" CC="$(CC)" CXX="$(CXX)"
 MAKE_COMMON = DESTDIR="" prefix=$(build_prefix) bindir=$(build_bindir) libdir=$(build_libdir) libexecdir=$(build_libexecdir) datarootdir=$(build_datarootdir) includedir=$(build_includedir) sysconfdir=$(build_sysconfdir)
 
 #autoconf configure-driven scripts: llvm pcre arpack fftw unwind gmp mpfr patchelf uv
-#custom Makefile rules: openlibm Rmath double-conversion dsfmt suitesparse-wrapper suitesparse lapack openblas utf8proc
+#custom Makefile rules: openlibm Rmath double-conversion dsfmt suitesparse-wrapper suitesparse lapack openblas mojibake
 
 # prevent installing libs into usr/lib64 on opensuse
 unexport CONFIG_SITE
@@ -111,8 +111,8 @@ ifeq ($(USE_SYSTEM_SUITESPARSE), 0)
 STAGE2_DEPS += suitesparse
 endif
 
-ifeq ($(USE_SYSTEM_UTF8PROC), 0)
-STAGE2_DEPS += utf8proc
+ifeq ($(USE_SYSTEM_MOJIBAKE), 0)
+STAGE2_DEPS += mojibake
 endif
 
 # Only compile standalone LAPACK if we are not using OpenBLAS.
@@ -144,7 +144,7 @@ install: $(addprefix install-, $(LIBS))
 cleanall: $(addprefix clean-, $(LIBS))
 distcleanall: $(addprefix distclean-, $(LIBS))
 	rm -rf $(build_prefix)
-getall: get-llvm get-uv get-pcre get-double-conversion get-openlibm get-openspecfun get-dsfmt get-Rmath get-openblas get-lapack get-fftw get-suitesparse get-arpack get-unwind get-osxunwind get-gmp get-mpfr get-zlib get-patchelf get-utf8proc
+getall: get-llvm get-uv get-pcre get-double-conversion get-openlibm get-openspecfun get-dsfmt get-Rmath get-openblas get-lapack get-fftw get-suitesparse get-arpack get-unwind get-osxunwind get-gmp get-mpfr get-zlib get-patchelf get-mojibake
 
 ## PATHS ##
 # sort is used to remove potential duplicates
@@ -1203,42 +1203,47 @@ compile-fftw-double: $(FFTW_DOUBLE_OBJ_TARGET)
 check-fftw-double: fftw-$(FFTW_VER)-double/checked
 install-fftw-double: $(FFTW_DOUBLE_OBJ_TARGET)
 
-## UTF8PROC ##
+## MOJIBAKE ##
 
-UTF8PROC_OBJ_SOURCE = utf8proc-v$(UTF8PROC_VER)/libutf8proc.a
-UTF8PROC_OBJ_LIB = $(build_libdir)/libutf8proc.a
-UTF8PROC_OBJ_HEADER = $(build_includedir)/utf8proc.h
-UTF8PROC_OBJ_TARGET = $(UTF8PROC_OBJ_LIB) $(UTF8PROC_OBJ_HEADER)
+MOJIBAKE_SRC_TARGET = libmojibake/libmojibake.a
+MOJIBAKE_OBJ_LIB    = $(build_libdir)/libmojibake.a
+MOJIBAKE_OBJ_HEADER = $(build_includedir)/mojibake.h
+MOJIBAKE_OBJ_TARGET = $(MOJIBAKE_OBJ_LIB) $(MOJIBAKE_OBJ_HEADER)
 
-utf8proc-v$(UTF8PROC_VER).tar.gz:
-	$(JLDOWNLOAD) $@ http://www.public-software-group.org/pub/projects/utf8proc/v$(UTF8PROC_VER)/$@
 
-utf8proc-v$(UTF8PROC_VER)/Makefile: utf8proc-v$(UTF8PROC_VER).tar.gz
-	$(JLCHECKSUM) $<
-	$(TAR) -xzf $<
-	patch $@ < utf8proc_Makefile.patch
-	cd utf8proc-v$(UTF8PROC_VER) && patch < ../utf8proc_msvc.patch
+libmojibake/Makefile:
+	(cd .. && git submodule init && git submodule update)
+ifeq (exists, $(shell [ -d libmojibake/.git ] && echo exists ))
+$(MOJIBAKE_SRC_TARGET): libmojibake/.git/HEAD
+endif
+ifeq (exists, $(shell [ -d $(JULIAHOME)/.git/modules/deps/libmojibake ] && echo exists ))
+$(MOJIBAKE_SRC_TARGET): $(JULIAHOME)/.git/modules/deps/libmojibake/HEAD
+endif
+$(MOJIBAKE_SRC_TARGET): libmojibake/Makefile
+	$(MAKE) -C libmojibake cc="$(CC) -O2 -std=c99 $(fPIC)" AR="$(AR)" libmojibake.a
 	touch -c $@
+libmojibake/checked: $(MOJIBAKE_SRC_TARGET)
+ifeq ($(OS),$(BUILD_OS))
+	-$(MAKE) -C libmojibake check
+endif
+	echo 1 > $@
 
-$(UTF8PROC_OBJ_SOURCE): utf8proc-v$(UTF8PROC_VER)/Makefile
-	$(MAKE) -C utf8proc-v$(UTF8PROC_VER) cc="$(CC) -O2 -std=c99 $(fPIC)" AR="$(AR)" libutf8proc.a
-
-$(UTF8PROC_OBJ_LIB): $(UTF8PROC_OBJ_SOURCE)
+$(MOJIBAKE_OBJ_LIB): $(MOJIBAKE_SRC_TARGET)
 	cp -f $< $@
 
-$(UTF8PROC_OBJ_HEADER): utf8proc-v$(UTF8PROC_VER)/Makefile
-	cp -f utf8proc-v$(UTF8PROC_VER)/utf8proc.h $@
+$(MOJIBAKE_OBJ_HEADER): libmojibake/Makefile
+	cp -f libmojibake/mojibake.h $@
 
-clean-utf8proc:
-	-$(MAKE) -C utf8proc-v$(UTF8PROC_VER) clean
-distclean-utf8proc: clean-utf8proc
-	-rm -rf utf8proc-v$(UTF8PROC_VER).tar.gz utf8proc-v$(UTF8PROC_VER)
+clean-mojibake:
+	-$(MAKE) -C libmojibake clean
+	-rm -rf $(build_libdir)/libmojibake.a $(build_includedir)/mojibake.h
+distclean-mojibake: clean-mojibake
 
-get-utf8proc: utf8proc-v$(UTF8PROC_VER).tar.gz
-configure-utf8proc: get-utf8proc
-compile-utf8proc: $(UTF8PROC_OBJ_SOURCE)
-check-utf8proc: 
-install-utf8proc: $(UTF8PROC_OBJ_TARGET)
+get-mojibake: libmojibake/Makefile
+configure-mojibake: get-mojibake
+compile-mojibake: $(MOJIBAKE_SRC_TARGET)
+check-mojibake: libmojibake/checked
+install-mojibake: $(MOJIBAKE_OBJ_TARGET)
 
 ## SUITESPARSE ##
 

deps/Versions.make

@@ -16,4 +16,3 @@ MPFR_VER=3.1.2
 ZLIB_VER = 1.2.8
 PATCHELF_VER = 0.8
 GIT_VER = 1.8.2.3
-UTF8PROC_VER = 1.1.6

src/Makefile

@@ -25,7 +25,7 @@ ifeq ($(USE_LLVM_SHLIB),1)
 LLVMLINK = $(call exec,$(LLVM_CONFIG) --ldflags) -lLLVM-$(call exec,$(LLVM_CONFIG) --version)
 endif
 
-COMMON_LIBS = -L$(build_shlibdir) -L$(build_libdir) $(LIBUV) $(LIBUTF8PROC) $(NO_WHOLE_ARCHIVE) $(LLVMLINK) $(OSLIBS)
+COMMON_LIBS = -L$(build_shlibdir) -L$(build_libdir) $(LIBUV) $(LIBMOJIBAKE) $(NO_WHOLE_ARCHIVE) $(LLVMLINK) $(OSLIBS)
 DEBUG_LIBS = $(WHOLE_ARCHIVE) $(JULIAHOME)/src/flisp/libflisp-debug.a $(WHOLE_ARCHIVE) $(JULIAHOME)/src/support/libsupport-debug.a $(COMMON_LIBS)
 RELEASE_LIBS = $(WHOLE_ARCHIVE) $(JULIAHOME)/src/flisp/libflisp.a $(WHOLE_ARCHIVE) $(JULIAHOME)/src/support/libsupport.a $(COMMON_LIBS)
 

src/flisp/Makefile

@@ -18,7 +18,7 @@ endif
 OBJS = $(SRCS:%.c=%.o)
 DOBJS = $(SRCS:%.c=%.do)
 LLTDIR = ../support
-LLT = $(LLTDIR)/libsupport.a $(LIBUV) $(LIBUTF8PROC)
+LLT = $(LLTDIR)/libsupport.a $(LIBUV) $(LIBMOJIBAKE)
 
 FLAGS = -I$(LLTDIR) $(CFLAGS) $(HFILEDIRS:%=-I%) \
         -I$(LIBUV_INC) -I$(build_includedir) $(LIBDIRS:%=-L%) \

src/flisp/Windows.mk

@@ -31,17 +31,17 @@ OBJECTS = \
 	basename.obj
 
 LIBUV = $(MAKEDIR)\..\..\deps\libuv\libuv.lib
-LIBUTF8PROC = $(MAKEDIR)\..\..\deps\utf8proc-v1.1.6\libutf8proc.lib
+LIBMOJIBAKE = $(MAKEDIR)\..\..\deps\libmojibake\libmojibake.lib
 LIBSUPPORT = $(MAKEDIR)\..\support\libsupport.lib
 
-INCLUDE = $(INCLUDE);$(MAKEDIR)\..\..\deps\libuv\include;$(MAKEDIR)\..\..\deps\utf8proc-v1.1.6;$(MAKEDIR)\..\support
+INCLUDE = $(INCLUDE);$(MAKEDIR)\..\..\deps\libuv\include;$(MAKEDIR)\..\..\deps\libmojibake;$(MAKEDIR)\..\support
 
 CFLAGS = $(CFLAGS) /Qstd=c99 -D_CRT_SECURE_NO_WARNINGS -DLIBRARY_EXPORTS
 LFLAGS = $(LFLAGS) kernel32.lib ws2_32.lib psapi.lib advapi32.lib iphlpapi.lib
 
 default: $(NAME).exe
 
-$(NAME).exe: lib$(NAME).lib flmain.obj $(LIBSUPPORT) $(LIBUV) $(LIBUTF8PROC)
+$(NAME).exe: lib$(NAME).lib flmain.obj $(LIBSUPPORT) $(LIBUV) $(LIBMOJIBAKE)
 	$(LINK) $(LFLAGS) /OUT:$(NAME).exe /PDB:$(NAME).pdb /MAP $** 
 
 $(LIBSUPPORT):
@@ -50,8 +50,8 @@ $(LIBSUPPORT):
 $(LIBUV):
 	PUSHD $(MAKEDIR)\..\..\deps\libuv && $(MAKE) /NOLOGO /F Windows.mk  && POPD
 
-$(LIBUTF8PROC):
-	PUSHD $(MAKEDIR)\..\..\deps\utf8proc-v1.1.6 && cl -nologo /c utf8proc.c && $(AR) /OUT:libutf8proc.lib utf8proc.obj && POPD
+$(LIBMOJIBAKE):
+	PUSHD $(MAKEDIR)\..\..\deps\libmojibake && cl -nologo /c utf8proc.c && $(AR) /OUT:libmojibake.lib utf8proc.obj && POPD
 
 lib$(NAME).lib: $(OBJECTS)
 	$(AR) /OUT:lib$(NAME).lib $**

src/flisp/julia_extensions.c

@@ -3,7 +3,7 @@
 #include <string.h>
 #include <assert.h>
 #include "flisp.h"
-#include "utf8proc.h"
+#include "mojibake.h"
 
 #ifdef __cplusplus
 extern "C" {

test/core.jl

@@ -1826,3 +1826,6 @@ sa = sub(a,4:6)
 try
     issue7897!(sa, zeros(10))
 end
+
+# issue #7582
+aₜ = "a variable using Unicode 6"