feat: 终端输入支持utf-8

CMakeLists.txt

@@ -82,6 +82,14 @@ if (WIN32 OR CYGWIN)
     list(APPEND base_compile_definitions aFunWIN32=1)
 endif()
 
+if (WIN32 AND NOT CYGWIN)
+    list(APPEND base_compile_definitions aFunWIN32_NO_CYGWIN=1)
+endif()
+
+if (CYGWIN)
+    list(APPEND base_compile_definitions aFunCYGWIN=1)
+endif()
+
 add_compile_definitions(${base_compile_definitions})
 
 include(${CMAKE_CURRENT_LIST_DIR}/deps/deps.cmake)  # 安装依赖

include/core/core_init.h

@@ -5,11 +5,6 @@
 #include <setjmp.h>
 #include "tool.h"
 
-#ifdef aFunWIN32
-#pragma warning(disable : 5105)
-#include "Windows.h"
-#endif
-
 AFUN_CORE_EXPORT extern Logger *aFunCoreLogger;
 AFUN_CORE_EXPORT bool aFunCoreInit(char *log_dir, LogFactoryPrintConsole print_console, bool fe, bool se, jmp_buf *buf, LogLevel level);
 

include/core/env.h

@@ -73,6 +73,8 @@ AFUN_CORE_EXPORT bool pushLiteralRegex(char *pattern, char *func, bool in_protec
 
 /* ErrorInfo 相关操作 */
 AFUN_CORE_EXPORT void fprintfErrorInfo(FILE *file, af_ErrorInfo *ei);
+AFUN_CORE_EXPORT void fprintfErrorInfoStderr(af_ErrorInfo *ei);
+AFUN_CORE_EXPORT void fprintfErrorInfoStdout(af_ErrorInfo *ei);
 
 /* ErrorBacktracking 相关操作 */
 AFUN_CORE_EXPORT void pushErrorBacktracking(FileLine line, FilePath file, char *note, af_ErrorInfo *ei);

include/core/parser.h

@@ -8,6 +8,8 @@
 #define SYNTACTIC_MAX_DEEP (1000)
 typedef struct af_Parser af_Parser;
 
+#define STDIN_MAX_SIZE (1024)
+
 /* Parser 创建与释放 */
 AFUN_CORE_EXPORT af_Parser *
 makeParser(DLC_SYMBOL(readerFunc) read_func, DLC_SYMBOL(destructReaderFunc) destruct_func, size_t data_size);

include/tool/stdio_.h

@@ -0,0 +1,26 @@
+#ifndef AFUN_STDIO__H
+#define AFUN_STDIO__H
+#include <stdio.h>
+
+AFUN_TOOL_EXPORT int fgets_stdin(char **dest, int len);
+
+#ifdef aFunWIN32_NO_CYGWIN
+AFUN_TOOL_EXPORT int fputs_stdout(char *str);
+AFUN_TOOL_EXPORT int fputs_stderr(char *str);
+
+AFUN_TOOL_EXPORT size_t vprintf_stdout(size_t buf_len, char *format, va_list ap);
+AFUN_TOOL_EXPORT size_t vprintf_stderr(size_t buf_len, char *format, va_list ap);
+AFUN_TOOL_EXPORT size_t printf_stdout(size_t buf_len, char *format, ...);
+AFUN_TOOL_EXPORT size_t printf_stderr(size_t buf_len, char *format, ...);
+
+#else
+#define fputs_stdout(str) fputs((str), stdout)
+#define fputs_stderr(str) fputs((str), stderr)
+
+#define vprintf_stdout(buf_len, format, ap) vfprintf(stdout, (format), (ap))
+#define vprintf_stderr(buf_len, format, ap) vfprintf(stderr, (format), (ap))
+#define printf_stdout(buf_len, ...) fprintf(stdout, __VA_ARGS__)
+#define printf_stderr(buf_len, ...) fprintf(stderr, __VA_ARGS__)
+
+#endif
+#endif //AFUN_STDIO__H

include/tool/tool.h

@@ -11,6 +11,7 @@
 #include "macro.h"
 #include "aFunToolExport.h"
 
+#include "stdio_.h"
 #include "btye.h"
 #include "dlc.h"
 #include "file.h"

src/core/code.c

@@ -465,7 +465,7 @@ char *codeToStr(af_Code *code, int n) {
 
 static void printLayerSpace(size_t layer) {
     for (size_t i = 0; i < layer; i++)
-        printf("    ");
+        fputs("    ", stdout);
 }
 
 void printCode(af_Code *bt) {

src/core/core_init.c

@@ -14,10 +14,6 @@ bool aFunCoreInit(char *log_dir, LogFactoryPrintConsole print_console, bool fe,
     getEndian();
     if (setlocale(LC_ALL, "") == NULL)
         return false;
-#ifdef aFunWIN32
-    if(!SetConsoleOutputCP(65001))  // 设置windows代码页为utf-8编码
-        return false;
-#endif
     if (log_dir == NULL)
         return false;
     char *log = strJoin(log_dir, "aFunlang", false, false);

src/core/env.c

@@ -60,6 +60,8 @@ static void freeAllErrorBacktracking(af_ErrorBacktracking *ebt);
 static char *getActivityInfoToBacktracking(af_Activity *activity);
 static char *getActivityTrackBackInfoToBacktracking(af_ActivityTrackBack *atb);
 static void fprintfNote(FILE *file, char *note);
+static void fprintfNoteStderr(char *note);
+static void fprintfNoteStdout(char *note);
 
 /* 内置顶层消息处理器 */
 static void mp_NORMAL(af_Message *msg, bool is_gc, af_Environment *env);
@@ -697,7 +699,7 @@ static void mp_ERROR(af_Message *msg, bool is_gc, af_Environment *env) {
         return;
     }
     if (!is_gc)
-        fprintfErrorInfo(stdout, *(af_ErrorInfo **)msg->msg);
+        fprintfErrorInfoStdout(*(af_ErrorInfo **)msg->msg);  // TODO-szh 获取EnvVar, 设定输出的位置 (stdout/stderr)
     freeErrorInfo(*(af_ErrorInfo **)msg->msg);
 }
 
@@ -1395,6 +1397,56 @@ void fprintfErrorInfo(FILE *file, af_ErrorInfo *ei) {
     fflush(file);
 }
 
+static void fprintfNoteStderr(char *note) {
+    char *ent = NULL;
+    while(true) {
+        ent = strchr(note, '\n');
+        if (ent != NULL)
+            *ent = NUL;
+        printf_stderr(0, "   #note %s\n", note);
+        if (ent == NULL)  // 意味着是最后一部分`note`
+            break;
+        *ent = '\n';
+        note = ent + 1;
+    }
+}
+
+void fprintfErrorInfoStderr(af_ErrorInfo *ei) {
+    printf_stderr(0, "Error Traceback (most recent call last):\n");
+    for (af_ErrorBacktracking *ebt = ei->track; ebt != NULL; ebt = ebt->next) {
+        printf_stderr(0, "  File \"%s\", line %d\n", ebt->file, ebt->line);
+        if (ebt->note != NULL)
+            fprintfNoteStderr(ebt->note);
+    }
+    printf_stderr(0, "%s: \"%s\"\n", ei->error_type, ei->error);
+    fflush(stderr);
+}
+
+static void fprintfNoteStdout(char *note) {
+    char *ent = NULL;
+    while(true) {
+        ent = strchr(note, '\n');
+        if (ent != NULL)
+            *ent = NUL;
+        printf_stdout(0, "   #note %s\n", note);
+        if (ent == NULL)  // 意味着是最后一部分`note`
+            break;
+        *ent = '\n';
+        note = ent + 1;
+    }
+}
+
+void fprintfErrorInfoStdout(af_ErrorInfo *ei) {
+    printf_stdout(0, "Error Traceback (most recent call last):\n");
+    for (af_ErrorBacktracking *ebt = ei->track; ebt != NULL; ebt = ebt->next) {
+        printf_stdout(0, "  File \"%s\", line %d\n", ebt->file, ebt->line);
+        if (ebt->note != NULL)
+            fprintfNoteStdout(ebt->note);
+    }
+    printf_stdout(0, "%s: \"%s\"\n", ei->error_type, ei->error);
+    fflush(stdout);
+}
+
 static af_ErrorBacktracking *makeErrorBacktracking(FileLine line, FilePath file, char *note) {
     af_ErrorBacktracking *ebt = calloc(1, sizeof(af_ErrorBacktracking));
     ebt->line = line;

src/core/lexical.c

@@ -7,6 +7,13 @@
 #include "__parser.h"
 #include "parserl_warning_error.h"
 
+#ifndef isascii
+#define isascii (((c) & ~0x7f) == 0)
+#endif
+
+#define isignore(ch) (isascii(ch) && (iscntrl(ch) || isspace(ch) || ch == ','))  /* 被忽略的符号 */
+#define iselement(ch) (!isascii(ch) || isgraph(ch))  /* 可以作为element的符号 */
+
 static void printLexicalError(char *info, af_Parser *parser) {
     writeErrorLog(aFunCoreLogger, log_default, "[Lexical] %s", info);
     parser->is_error = true;
@@ -47,9 +54,9 @@ static void setLexicalLast(af_LexicalStatus status, af_TokenType token, af_Parse
  *     -> ] -> [lex_rb] # return -1
  *     -> } -> [lex_rc] # return -1
  *     -> ; -> (lex_comment_before)
- *     -> iscntrl(ch) || isspace(ch) || , -> [lex_space]
+ *     -> isignore(ch) -> [lex_space]
  *     -> | -> (lex_element_long)
- *     -> isgraph(ch) -> [lex_element]
+ *     -> iselement(ch) -> [lex_element]
  */
 
 static int doneBegin(char ch, af_Parser *parser) {
@@ -101,13 +108,13 @@ static int doneBegin(char ch, af_Parser *parser) {
     } else if (ch == ';') {
         parser->lexical->status = lex_comment_before;
         return 1;
-    } else if (iscntrl(ch) || isspace(ch) || ch == ',') {  // 空白符或控制字符被忽略
+    } else if (isignore(ch)) {  // 空白符或控制字符被忽略
         setLexicalLast(lex_space, TK_SPACE, parser);
         return 1;
     } else if (ch == '|') {
         parser->lexical->status = lex_element_long;
         return 1;
-    } else if (isgraph(ch)) {  // 除空格外的可见字符
+    } else if (iselement(ch)) {  // 除空格外的可见字符
         setLexicalLast(lex_element_short, TK_ELEMENT_SHORT, parser);
         return 1;
     }
@@ -284,11 +291,11 @@ static int doneElementLongEnd(char ch, af_Parser *parser) {
 /*
  * 状态机图:
  * [lex_element_short]
- *      -> !strchr("!@#([{}]);,", ch) && isgraph(ch) -> (lex_element_short)
+ *      -> !strchr("!@#([{}]);,", ch) && iselement(ch) -> (lex_element_short)
  *      -> other -> (lex_element_short) # return -1
  */
 static int doneElementShort(char ch, af_Parser *parser) {
-    if (!strchr("!@#([{}]);,", ch) && isgraph(ch)) {  // 除空格外的可见字符 (不包括NUL)
+    if (!strchr("!@#([{}]);,", ch) && iselement(ch)) {  // 除空格外的可见字符 (不包括NUL)
         setLexicalLast(lex_element_short, TK_ELEMENT_SHORT, parser);
         return 1;
     }
@@ -299,11 +306,11 @@ static int doneElementShort(char ch, af_Parser *parser) {
 /*
  * 状态机图:
  * [lex_space]
- *      -> ch != NUL && (iscntrl(ch) || isspace(ch)) || , -> (lex_space)
+ *      -> ch != NUL && isignore(ch) -> (lex_space)
  *      -> other -> (lex_space) # return -1
  */
 static int doneSpace(char ch, af_Parser *parser) {
-    if (ch != NUL && (iscntrl(ch) || isspace(ch)) || ch == ',') {
+    if (ch != NUL && isignore(ch)) {
         setLexicalLast(lex_space, TK_SPACE, parser);
         return 1;
     }
@@ -331,7 +338,7 @@ af_TokenType getTokenFromLexical(char **text, af_Parser *parser) {
 
     while (1) {
         char ch = getChar(parser->reader);
-        if (iscntrl(ch) && !isspace(ch) && ch != NUL)
+        if (isascii(ch) && iscntrl(ch) && !isspace(ch) && ch != NUL)  // ascii 控制字符
             printLexicalWarning(INCULDE_CONTROL(base), parser);
 
         switch (parser->lexical->status) {

src/core/parser.c

@@ -5,7 +5,6 @@
 
 #include "aFunCore.h"
 #include "__parser.h"
-#include <errno.h>
 
 static af_Lexical *makeLexical(void);
 static void freeLexical(af_Lexical *lex);
@@ -109,7 +108,7 @@ static void destructFile(struct readerDataFile *data) {
 af_Parser *makeParserByFile(FilePath path){
     FILE *file = fopen(path, "rb");
     if (file == NULL) {
-        writeErrorLog(aFunCoreLogger, log_default, "File open error: %s", strerror(errno));
+        writeErrorLog(aFunCoreLogger, log_default, "File open error: %s", file);
         return NULL;
     }
 
@@ -125,42 +124,50 @@ af_Parser *makeParserByFile(FilePath path){
 
 struct readerDataStdin {
     bool no_first;
-    bool read_continue;  /* 上一次的内容没有读取完毕 */
+
+    char *data;
+    size_t index;
+    size_t len;
 };
 
 static size_t readFuncStdin(struct readerDataStdin *data, char *dest, size_t len, bool *read_end) {
-    if (!data->read_continue) {
+    if (data->index == data->len) {  // 读取内容
         if (data->no_first)
             printf("... ");
         else
             printf(">>> ");
+        data->no_first = true;
+
+        free(data->data);
 
         int ch = getc(stdin);
         if (ch == '\n' || ch == EOF) {
             /* 读取结束 */
             *read_end = true;
-            return false;
+            return 0;
         }
 
         ungetc(ch, stdin);
-    }
 
-    data->no_first = false;
-    data->read_continue = false;
+        if (fgets_stdin(&data->data, STDIN_MAX_SIZE) == 0) {
+            writeErrorLog(aFunCoreLogger, log_default, "The stdin buf too large (> %d)", STDIN_MAX_SIZE);
+            *read_end = true;
+            return 0;
+        }
 
-    if (fgets(dest, (int)(len + 1), stdin) == NULL) {  // + 1 是因为len不包含NUL的位置
-        *read_end = true;
-        return 0;
+        data->index = 0;
+        data->len = strlen(data->data);
     }
 
-    if (strchr(dest, '\n') == NULL)  // 未找到 \n 代表还没读取完
-        data->read_continue = true;
-
-    return strlen(dest);
+    if (data->index + len > data->len)  // 超出长度范围
+        len = data->len - data->index;
+    memcpy(dest, data->data + data->index, len);
+    data->index += len;
+    return len;
 }
 
 static void destructStdin(struct readerDataStdin *data) {
-    // 什么都不用做
+    free(data->data);
 }
 
 af_Parser *makeParserByStdin(){

src/main.c

@@ -66,7 +66,7 @@ int main(int argc, char **argv) {
 
     if (!re) {
 INIT_ERROR:
-        fprintf(stderr, "aFunlang init error.");
+        printf_stderr(0, "aFunlang init error.");
         return EXIT_FAILURE;
     }
 
@@ -97,20 +97,20 @@ INIT_ERROR:
 }
 
 static void printVersion(void) {
-    printf("aFunlang at %s\n", name);
-    printf("version: " aFunVersion "\n");
-    printf(aFunDescription "\n");
+    printf_stdout(0, "aFunlang at %s\n", name);
+    printf_stdout(0, "version: " aFunVersion "\n");
+    printf_stdout(strlen(aFunDescription), aFunDescription "\n");
 }
 
 static void printWelcomeInfo(void) {
-    printf("\naFunlang " aFunVersion " CommandLine (" __DATE__ ", " __TIME__ ")\n");
-    printf("["compilerID"] on "systemName"\n");
-    printf("(Enter the aFun code to run in the top activity)\n");
+    printf_stdout(0, "\naFunlang " aFunVersion " CommandLine (" __DATE__ ", " __TIME__ ")\n");
+    printf_stdout(0, "["compilerID"] on "systemName"\n");
+    printf_stdout(0, "(Enter the aFun code to run in the top activity)\n");
 }
 
 static void printHelp(void) {
-    printf("aFunlang Usage:\n");
-    printf("%s\n", help_info);
+    printf_stdout(0, "aFunlang Usage:\n");
+    printf_stdout(strlen(help_info), "%s\n", help_info);
 }
 
 /*

src/runtime/base/str_obj.c

@@ -26,6 +26,7 @@ static void strDestruct(char *id, af_Object *obj, ObjectString *data, af_Environ
 static void strLiteral(char *id, af_Object *obj, ObjectString *data, char *str, af_Environment *env) {
     if (!EQ_STR(id, string_id) || data->str != NULL)
         return;
+    writeDebugLog(aFunCoreLogger, log_default, "strLiteral str = %s, %d", str, strlen(str));
     data->str = NEW_STR(STR_LEN(str) - 2);  // 取出两个引号
     memcpy(data->str, str + 1, (STR_LEN(str) - 2) * sizeof(char));
 }

src/tool/log.c

@@ -6,6 +6,7 @@
  * time_s.h 中 getTime -> strCopy
  * mem.h 中 free
  * file.h 中 getFileSize
+ * stdio_.h
  */
 
 #include <stdio.h>
@@ -17,11 +18,13 @@
 #include "log.h"
 #include "time_s.h"
 #include "file.h"
+#include "stdio_.h"
 
-#if aFunWIN32
+#ifdef aFunWIN32
 #include <windows.h>
 #define getpid() (long)GetCurrentProcessId()
 #define gettid() (long)GetCurrentThreadId()
+// cygwin没有syscall.h, 因此需要依赖 windows 的 api
 #else
 #include <unistd.h>
 #include "sys/syscall.h"
@@ -151,8 +154,7 @@ static const char *LogLevelNameLong[] = {
         "*FATAL ERROR*",  // fatal_error 5
 };
 
-static int writeLog_(Logger *logger, LogLoggerPrintConsole pc, LogLevel level, char *file, int line, char *func, char *format,
-          va_list ap){
+static int writeLog_(Logger *logger, LogLoggerPrintConsole pc, LogLevel level, char *file, int line, char *func, char *format, va_list ap){
     if (logger->level > level)
         return 2;
     if (!log_factory.init || log_factory.log == NULL)
@@ -182,26 +184,27 @@ static int writeLog_(Logger *logger, LogLoggerPrintConsole pc, LogLevel level, c
         fflush(log_factory.csv);
     }
 
+#define STD_BUF_SIZE (strlen(tmp) + 1024)
     if (pc != log_print_no_console) {
         switch (log_factory.print_console) {
             case log_pc_all:
                 if (level < log_warning) {
-                    fprintf(stdout, FORMAT_SHORT, LogLevelNameLong[level], logger->id, file, line, func, tmp);
+                    printf_stdout(STD_BUF_SIZE, FORMAT_SHORT, LogLevelNameLong[level], logger->id, file, line, func, tmp);
                     fflush(stdout);
                 } else if (log_factory.print_console) {
-                    fprintf(stderr, FORMAT_SHORT, LogLevelNameLong[level], logger->id, file, line, func, tmp);
+                    printf_stderr(STD_BUF_SIZE, FORMAT_SHORT, LogLevelNameLong[level], logger->id, file, line, func, tmp);
                     fflush(stderr);
                 }
                 break;
             case log_pc_w:
                 if (level >= log_warning) { // warning的内容一定会被打印
-                    fprintf(stderr, FORMAT_SHORT, LogLevelNameLong[level], logger->id, file, line, func, tmp);
+                    printf_stderr(STD_BUF_SIZE, FORMAT_SHORT, LogLevelNameLong[level], logger->id, file, line, func, tmp);
                     fflush(stderr);
                 }
                 break;
             case log_pc_e:
                 if (level >= log_error) {  // warning的内容一定会被打印
-                    fprintf(stderr, FORMAT_SHORT, LogLevelNameLong[level], logger->id, file, line, func, tmp);
+                    printf_stderr(STD_BUF_SIZE, FORMAT_SHORT, LogLevelNameLong[level], logger->id, file, line, func, tmp);
                     fflush(stderr);
                 }
                 break;
@@ -211,6 +214,7 @@ static int writeLog_(Logger *logger, LogLoggerPrintConsole pc, LogLevel level, c
                 break;
         }
     }
+#undef STD_BUF_SIZE
 
     free(ti);
 #undef FORMAT

src/tool/stdio_.c

@@ -0,0 +1,117 @@
+/*
+ * 文件名: stdio_.c
+ * 目标: 用于终端输出的控制 (stdout, stdin, stderr)
+ * 因为不同平台上终端采用的编码不同
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdarg.h>
+#include "tool.h"
+
+#ifdef aFunWIN32_NO_CYGWIN
+#include <Windows.h>
+// 获取CodePage, 并将内存中utf-8字符串转换为对应编码输出
+// cygwin环境下, 终端默认为uft-8
+
+static int convertMultiByte(char **dest, char *str, UINT from, UINT to) {
+    if (str == NULL || dest == NULL)
+        return 0;
+
+    int tmp_len = MultiByteToWideChar(from, 0, str, -1, 0, 0);
+    if (tmp_len == 0)
+        return 0;
+
+    wchar_t *tmp = calloc(tmp_len + 1, sizeof(wchar_t));
+    if (MultiByteToWideChar(from, 0, str, -1, tmp, tmp_len) == 0)
+        return 0;
+
+    int dest_len = WideCharToMultiByte(to, 0, tmp, -1, NULL, 0, NULL, NULL);
+    if (dest_len == 0)
+        return 0;
+
+    *dest = calloc(dest_len + 1, sizeof(char));
+    int re = WideCharToMultiByte(to, 0, tmp, -1, *dest, dest_len, NULL, NULL);
+
+    free(tmp);
+    return re;
+}
+
+int fgets_stdin(char **dest, int len) {
+    char *wstr = calloc(len, sizeof(char));
+    int re = 0;
+
+    UINT code_page = GetConsoleCP();
+    if (fgets(wstr, len, stdin) != NULL)
+        re = convertMultiByte(dest, wstr, code_page, CP_UTF8);
+    return re;
+}
+
+static int fputs_std_(char *str, FILE *std) {
+    UINT code_page = GetConsoleCP();
+    char *wstr = NULL;
+    int re = EOF;
+    convertMultiByte(&wstr, str, CP_UTF8, code_page);
+    if (wstr != NULL) {
+        re = fputs(wstr, std);
+        free(wstr);
+    }
+    return re;
+}
+
+int fputs_stdout(char *str) {
+    return fputs_std_(str, stdout);
+}
+
+int fputs_stderr(char *str) {
+    return fputs_std_(str, stderr);
+}
+
+static size_t vprintf_std_(FILE *std, size_t buf_len, char *format, va_list ap) {
+    if (buf_len == 0)
+        buf_len = 1024;
+    buf_len += 10;  // 预留更多位置
+    char *buf = calloc(buf_len, sizeof(char));
+    size_t re = vsnprintf(buf, buf_len, format, ap);
+    if (fputs_std_(buf, std) == EOF)
+        re = 0;
+    free(buf);
+    return re;
+}
+
+size_t vprintf_stdout(size_t buf_len, char *format, va_list ap) {
+    return vprintf_std_(stdout, buf_len, format, ap);
+}
+
+size_t vprintf_stderr(size_t buf_len, char *format, va_list ap) {
+    return vprintf_std_(stderr, buf_len, format, ap);
+}
+
+size_t printf_stdout(size_t buf_len, char *format, ...) {
+    va_list ap;
+    va_start(ap, format);
+    size_t re = vprintf_std_(stdout, buf_len, format, ap);
+    va_end(ap);
+    return re;
+}
+
+size_t printf_stderr(size_t buf_len, char *format, ...) {
+    va_list ap;
+    va_start(ap, format);
+    size_t re = vprintf_std_(stderr, buf_len, format, ap);
+    va_end(ap);
+    return re;
+}
+
+#else
+// 用于Linux平台的IO函数
+// 默认Linux平台均使用utf-8
+
+int fgets_stdin(char **dest, int len) {
+    *dest = calloc(len, sizeof(char));
+    if (fgets(*dest, len, stdin) == NULL)
+        return 0;
+    return 1;
+}
+
+#endif