You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1954 lines
54 KiB
1954 lines
54 KiB
/**
|
|
*** Build a deterministic finite automaton to associate CCSIDs with
|
|
*** character set names.
|
|
***
|
|
*** Compile on OS/400 with options SYSIFCOPT(*IFSIO).
|
|
***
|
|
*** See Copyright for the status of this software.
|
|
***
|
|
*** Author: Patrick Monnerat <pm@datasphere.ch>, DATASPHERE S.A.
|
|
**/
|
|
|
|
#include <stdio.h>
|
|
#include <errno.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <fcntl.h>
|
|
#include <ctype.h>
|
|
|
|
#include <iconv.h>
|
|
|
|
|
|
#ifdef OLDXML
|
|
#include "xml.h"
|
|
#else
|
|
#include <libxml/hash.h>
|
|
#include <libxml/parser.h>
|
|
#include <libxml/xpath.h>
|
|
#include <libxml/xpathInternals.h>
|
|
#endif
|
|
|
|
|
|
#ifdef __OS400__
|
|
#define iconv_open_error(cd) ((cd).return_value == -1)
|
|
#define set_iconv_open_error(cd) ((cd).return_value = -1)
|
|
#else
|
|
#define iconv_open_error(cd) ((cd) == (iconv_t) -1)
|
|
#define set_iconv_open_error(cd) ((cd) = (iconv_t) -1)
|
|
#endif
|
|
|
|
|
|
#define C_SOURCE_CCSID 500
|
|
#define C_UTF8_CCSID 1208
|
|
|
|
|
|
#define UTF8_SPACE 0x20
|
|
#define UTF8_HT 0x09
|
|
#define UTF8_0 0x30
|
|
#define UTF8_9 0x39
|
|
#define UTF8_A 0x41
|
|
#define UTF8_Z 0x5A
|
|
#define UTF8_a 0x61
|
|
#define UTF8_z 0x7A
|
|
|
|
|
|
#define GRANULE 128 /* Memory allocation granule. */
|
|
|
|
#define EPSILON 0x100 /* Token for empty transition. */
|
|
|
|
|
|
#ifndef OFFSETOF
|
|
#define OFFSETOF(t, f) ((unsigned int) ((char *) &((t *) 0)->f - (char *) 0))
|
|
#endif
|
|
|
|
#ifndef OFFSETBY
|
|
#define OFFSETBY(t, p, o) ((t *) ((char *) (p) + (unsigned int) (o)))
|
|
#endif
|
|
|
|
|
|
typedef struct t_transition t_transition; /* NFA/DFA transition. */
|
|
typedef struct t_state t_state; /* NFA/DFA state node. */
|
|
typedef struct t_symlist t_symlist; /* Symbol (i.e.: name) list. */
|
|
typedef struct t_chset t_chset; /* Character set. */
|
|
typedef struct t_stategroup t_stategroup; /* Optimization group. */
|
|
typedef unsigned char utf8char; /* UTF-8 character byte. */
|
|
typedef unsigned char byte; /* Untyped data byte. */
|
|
|
|
|
|
typedef struct { /* Set of pointers. */
|
|
unsigned int p_size; /* Current allocated size. */
|
|
unsigned int p_card; /* Current element count. */
|
|
void * p_set[1]; /* Element array. */
|
|
} t_powerset;
|
|
|
|
|
|
struct t_transition {
|
|
t_transition * t_forwprev; /* Head of forward transition list. */
|
|
t_transition * t_forwnext; /* Tail of forward transition list. */
|
|
t_transition * t_backprev; /* Head of backward transition list. */
|
|
t_transition * t_backnext; /* Tail of backward transition list. */
|
|
t_state * t_from; /* Incoming state. */
|
|
t_state * t_to; /* Destination state. */
|
|
unsigned short t_token; /* Transition token. */
|
|
unsigned int t_index; /* Transition array index. */
|
|
};
|
|
|
|
|
|
struct t_state {
|
|
t_state * s_next; /* Next state (for DFA construction). */
|
|
t_state * s_stack; /* Unprocessed DFA states stack. */
|
|
t_transition * s_forward; /* Forward transitions. */
|
|
t_transition * s_backward; /* Backward transitions. */
|
|
t_chset * s_final; /* Recognized character set. */
|
|
t_powerset * s_nfastates; /* Corresponding NFA states. */
|
|
unsigned int s_index; /* State index. */
|
|
};
|
|
|
|
|
|
struct t_symlist {
|
|
t_symlist * l_next; /* Next name in list. */
|
|
utf8char l_symbol[1]; /* Name bytes. */
|
|
};
|
|
|
|
|
|
struct t_chset {
|
|
t_chset * c_next; /* Next character set. */
|
|
t_symlist * c_names; /* Character set name list. */
|
|
iconv_t c_fromUTF8; /* Conversion from UTF-8. */
|
|
unsigned int c_ccsid; /* IBM character set code. */
|
|
unsigned int c_mibenum; /* IANA character code. */
|
|
};
|
|
|
|
|
|
struct t_stategroup {
|
|
t_stategroup * g_next; /* Next group. */
|
|
t_state * g_member; /* Group member (s_stack) list. */
|
|
unsigned int g_id; /* Group ident. */
|
|
};
|
|
|
|
|
|
|
|
t_chset * chset_list; /* Character set list. */
|
|
t_state * initial_state; /* Initial NFA state. */
|
|
iconv_t job2utf8; /* Job CCSID to UTF-8 conversion. */
|
|
iconv_t utf82job; /* UTF-8 to job CCSID conversion. */
|
|
t_state * dfa_states; /* List of DFA states. */
|
|
unsigned int groupid; /* Group ident counter. */
|
|
|
|
|
|
/**
|
|
*** UTF-8 strings.
|
|
**/
|
|
|
|
#pragma convert(819)
|
|
|
|
static const utf8char utf8_MIBenum[] = "MIBenum";
|
|
static const utf8char utf8_mibenum[] = "mibenum";
|
|
static const utf8char utf8_ibm_[] = "ibm-";
|
|
static const utf8char utf8_IBMCCSID[] = "IBMCCSID";
|
|
static const utf8char utf8_iana_[] = "iana-";
|
|
static const utf8char utf8_Name[] = "Name";
|
|
static const utf8char utf8_Pref_MIME_Name[] = "Preferred MIME Name";
|
|
static const utf8char utf8_Aliases[] = "Aliases";
|
|
static const utf8char utf8_html[] = "html";
|
|
static const utf8char utf8_htmluri[] = "http://www.w3.org/1999/xhtml";
|
|
static const utf8char utf8_A[] = "A";
|
|
static const utf8char utf8_C[] = "C";
|
|
static const utf8char utf8_M[] = "M";
|
|
static const utf8char utf8_N[] = "N";
|
|
static const utf8char utf8_P[] = "P";
|
|
static const utf8char utf8_T[] = "T";
|
|
static const utf8char utf8_ccsid[] = "ccsid";
|
|
static const utf8char utf8_EBCDIC[] = "EBCDIC";
|
|
static const utf8char utf8_ASCII[] = "ASCII";
|
|
static const utf8char utf8_assocnodes[] = "/ccsid_mibenum/assoc[@ccsid]";
|
|
static const utf8char utf8_aliastext[] =
|
|
"/ccsid_mibenum/assoc[@ccsid=$C]/alias/text()";
|
|
#ifdef OLDXML
|
|
static const utf8char utf8_tablerows[] =
|
|
"//table[@id='table-character-sets-1']/*/tr";
|
|
static const utf8char utf8_headerpos[] =
|
|
"count(th[text()=$T]/preceding-sibling::th)+1";
|
|
static const utf8char utf8_getmibenum[] = "number(td[$M])";
|
|
static const utf8char utf8_getprefname[] = "string(td[$P])";
|
|
static const utf8char utf8_getname[] = "string(td[$N])";
|
|
static const utf8char utf8_getaliases[] = "td[$A]/text()";
|
|
#else
|
|
static const utf8char utf8_tablerows[] =
|
|
"//html:table[@id='table-character-sets-1']/*/html:tr";
|
|
static const utf8char utf8_headerpos[] =
|
|
"count(html:th[text()=$T]/preceding-sibling::html:th)+1";
|
|
static const utf8char utf8_getmibenum[] = "number(html:td[$M])";
|
|
static const utf8char utf8_getprefname[] = "string(html:td[$P])";
|
|
static const utf8char utf8_getname[] = "string(html:td[$N])";
|
|
static const utf8char utf8_getaliases[] = "html:td[$A]/text()";
|
|
#endif
|
|
|
|
#pragma convert(0)
|
|
|
|
|
|
/**
|
|
*** UTF-8 character length table.
|
|
***
|
|
*** Index is first character byte, value is the character byte count.
|
|
**/
|
|
|
|
static signed char utf8_chlen[] = {
|
|
/* 00-07 */ 1, 1, 1, 1, 1, 1, 1, 1,
|
|
/* 08-0F */ 1, 1, 1, 1, 1, 1, 1, 1,
|
|
/* 10-17 */ 1, 1, 1, 1, 1, 1, 1, 1,
|
|
/* 18-1F */ 1, 1, 1, 1, 1, 1, 1, 1,
|
|
/* 20-27 */ 1, 1, 1, 1, 1, 1, 1, 1,
|
|
/* 28-2F */ 1, 1, 1, 1, 1, 1, 1, 1,
|
|
/* 30-37 */ 1, 1, 1, 1, 1, 1, 1, 1,
|
|
/* 38-3F */ 1, 1, 1, 1, 1, 1, 1, 1,
|
|
/* 40-47 */ 1, 1, 1, 1, 1, 1, 1, 1,
|
|
/* 48-4F */ 1, 1, 1, 1, 1, 1, 1, 1,
|
|
/* 50-57 */ 1, 1, 1, 1, 1, 1, 1, 1,
|
|
/* 58-5F */ 1, 1, 1, 1, 1, 1, 1, 1,
|
|
/* 60-67 */ 1, 1, 1, 1, 1, 1, 1, 1,
|
|
/* 68-6F */ 1, 1, 1, 1, 1, 1, 1, 1,
|
|
/* 70-77 */ 1, 1, 1, 1, 1, 1, 1, 1,
|
|
/* 78-7F */ 1, 1, 1, 1, 1, 1, 1, 1,
|
|
/* 80-87 */ -1, -1, -1, -1, -1, -1, -1, -1,
|
|
/* 88-8F */ -1, -1, -1, -1, -1, -1, -1, -1,
|
|
/* 90-97 */ -1, -1, -1, -1, -1, -1, -1, -1,
|
|
/* 98-9F */ -1, -1, -1, -1, -1, -1, -1, -1,
|
|
/* A0-A7 */ -1, -1, -1, -1, -1, -1, -1, -1,
|
|
/* A8-AF */ -1, -1, -1, -1, -1, -1, -1, -1,
|
|
/* B0-B7 */ -1, -1, -1, -1, -1, -1, -1, -1,
|
|
/* B8-BF */ -1, -1, -1, -1, -1, -1, -1, -1,
|
|
/* C0-C7 */ 2, 2, 2, 2, 2, 2, 2, 2,
|
|
/* C8-CF */ 2, 2, 2, 2, 2, 2, 2, 2,
|
|
/* D0-D7 */ 2, 2, 2, 2, 2, 2, 2, 2,
|
|
/* D8-DF */ 2, 2, 2, 2, 2, 2, 2, 2,
|
|
/* E0-E7 */ 3, 3, 3, 3, 3, 3, 3, 3,
|
|
/* E8-EF */ 3, 3, 3, 3, 3, 3, 3, 3,
|
|
/* F0-F7 */ 4, 4, 4, 4, 4, 4, 4, 4,
|
|
/* F8-FF */ 5, 5, 5, 5, 6, 6, -1, -1
|
|
};
|
|
|
|
|
|
|
|
void
|
|
chknull(void * p)
|
|
|
|
{
|
|
if (p)
|
|
return;
|
|
|
|
fprintf(stderr, "Not enough memory\n");
|
|
exit(1);
|
|
}
|
|
|
|
|
|
void
|
|
makecode(char * buf, unsigned int ccsid)
|
|
|
|
{
|
|
ccsid &= 0xFFFF;
|
|
memset(buf, 0, 32);
|
|
sprintf(buf, "IBMCCSID%05u0000000", ccsid);
|
|
}
|
|
|
|
|
|
iconv_t
|
|
iconv_open_ccsid(unsigned int ccsidout,
|
|
unsigned int ccsidin, unsigned int nullflag)
|
|
|
|
{
|
|
char fromcode[33];
|
|
char tocode[33];
|
|
|
|
makecode(fromcode, ccsidin);
|
|
makecode(tocode, ccsidout);
|
|
memset(tocode + 13, 0, sizeof tocode - 13);
|
|
|
|
if (nullflag)
|
|
fromcode[18] = '1';
|
|
|
|
return iconv_open(tocode, fromcode);
|
|
}
|
|
|
|
|
|
unsigned int
|
|
getnum(char * * cpp)
|
|
|
|
{
|
|
unsigned int n;
|
|
char * cp;
|
|
|
|
cp = *cpp;
|
|
n = 0;
|
|
|
|
while (isdigit(*cp))
|
|
n = 10 * n + *cp++ - '0';
|
|
|
|
*cpp = cp;
|
|
return n;
|
|
}
|
|
|
|
|
|
const utf8char *
|
|
hashBinaryKey(const byte * bytes, unsigned int len)
|
|
|
|
{
|
|
const byte * bp;
|
|
utf8char * key;
|
|
utf8char * cp;
|
|
unsigned int n;
|
|
unsigned int n4;
|
|
unsigned int i;
|
|
|
|
/**
|
|
*** Encode binary data in character form to be used as hash
|
|
*** table key.
|
|
**/
|
|
|
|
n = (4 * len + 2) / 3;
|
|
key = (utf8char *) malloc(n + 1);
|
|
chknull(key);
|
|
bp = bytes;
|
|
cp = key;
|
|
|
|
for (n4 = n >> 2; n4; n4--) {
|
|
i = (bp[0] << 16) | (bp[1] << 8) | bp[2];
|
|
*cp++ = 0x21 + ((i >> 18) & 0x3F);
|
|
*cp++ = 0x21 + ((i >> 12) & 0x3F);
|
|
*cp++ = 0x21 + ((i >> 6) & 0x3F);
|
|
*cp++ = 0x21 + (i & 0x3F);
|
|
bp += 3;
|
|
}
|
|
|
|
switch (n & 0x3) {
|
|
|
|
case 2:
|
|
*cp++ = 0x21 + ((*bp >> 2) & 0x3F);
|
|
*cp++ = 0x21 + ((*bp << 4) & 0x3F);
|
|
break;
|
|
|
|
case 3:
|
|
i = (bp[0] << 8) | bp[1];
|
|
*cp++ = 0x21 + ((i >> 10) & 0x3F);
|
|
*cp++ = 0x21 + ((i >> 4) & 0x3F);
|
|
*cp++ = 0x21 + ((i << 2) & 0x3F);
|
|
break;
|
|
}
|
|
|
|
*cp = '\0';
|
|
return key;
|
|
}
|
|
|
|
|
|
void *
|
|
hash_get(xmlHashTablePtr h, const void * binkey, unsigned int len)
|
|
|
|
{
|
|
const utf8char * key;
|
|
void * result;
|
|
|
|
key = hashBinaryKey((const byte *) binkey, len);
|
|
result = xmlHashLookup(h, key);
|
|
free((char *) key);
|
|
return result;
|
|
}
|
|
|
|
|
|
int
|
|
hash_add(xmlHashTablePtr h, const void * binkey, unsigned int len, void * data)
|
|
|
|
{
|
|
const utf8char * key;
|
|
int result;
|
|
|
|
key = hashBinaryKey((const byte *) binkey, len);
|
|
result = xmlHashAddEntry(h, key, data);
|
|
free((char *) key);
|
|
return result;
|
|
}
|
|
|
|
|
|
xmlDocPtr
|
|
loadXMLFile(const char * filename)
|
|
|
|
{
|
|
struct stat sbuf;
|
|
byte * databuf;
|
|
int fd;
|
|
int i;
|
|
xmlDocPtr doc;
|
|
|
|
if (stat(filename, &sbuf))
|
|
return (xmlDocPtr) NULL;
|
|
|
|
databuf = malloc(sbuf.st_size + 4);
|
|
|
|
if (!databuf)
|
|
return (xmlDocPtr) NULL;
|
|
|
|
fd = open(filename, O_RDONLY
|
|
#ifdef O_BINARY
|
|
| O_BINARY
|
|
#endif
|
|
);
|
|
|
|
if (fd < 0) {
|
|
free((char *) databuf);
|
|
return (xmlDocPtr) NULL;
|
|
}
|
|
|
|
i = read(fd, (char *) databuf, sbuf.st_size);
|
|
close(fd);
|
|
|
|
if (i != sbuf.st_size) {
|
|
free((char *) databuf);
|
|
return (xmlDocPtr) NULL;
|
|
}
|
|
|
|
databuf[i] = databuf[i + 1] = databuf[i + 2] = databuf[i + 3] = 0;
|
|
doc = xmlParseMemory((xmlChar *) databuf, i);
|
|
free((char *) databuf);
|
|
return doc;
|
|
}
|
|
|
|
|
|
int
|
|
match(char * * cpp, char * s)
|
|
|
|
{
|
|
char * cp;
|
|
int c1;
|
|
int c2;
|
|
|
|
cp = *cpp;
|
|
|
|
for (cp = *cpp; c2 = *s++; cp++) {
|
|
c1 = *cp;
|
|
|
|
if (c1 != c2) {
|
|
if (isupper(c1))
|
|
c1 = tolower(c1);
|
|
|
|
if (isupper(c2))
|
|
c2 = tolower(c2);
|
|
}
|
|
|
|
if (c1 != c2)
|
|
return 0;
|
|
}
|
|
|
|
c1 = *cp;
|
|
|
|
while (c1 == ' ' || c1 == '\t')
|
|
c1 = *++cp;
|
|
|
|
*cpp = cp;
|
|
return 1;
|
|
}
|
|
|
|
|
|
t_state *
|
|
newstate(void)
|
|
|
|
{
|
|
t_state * s;
|
|
|
|
s = (t_state *) malloc(sizeof *s);
|
|
chknull(s);
|
|
memset((char *) s, 0, sizeof *s);
|
|
return s;
|
|
}
|
|
|
|
|
|
void
|
|
unlink_transition(t_transition * t)
|
|
|
|
{
|
|
if (t->t_backnext)
|
|
t->t_backnext->t_backprev = t->t_backprev;
|
|
|
|
if (t->t_backprev)
|
|
t->t_backprev->t_backnext = t->t_backnext;
|
|
else if (t->t_to)
|
|
t->t_to->s_backward = t->t_backnext;
|
|
|
|
if (t->t_forwnext)
|
|
t->t_forwnext->t_forwprev = t->t_forwprev;
|
|
|
|
if (t->t_forwprev)
|
|
t->t_forwprev->t_forwnext = t->t_forwnext;
|
|
else if (t->t_from)
|
|
t->t_from->s_forward = t->t_forwnext;
|
|
|
|
t->t_backprev = (t_transition *) NULL;
|
|
t->t_backnext = (t_transition *) NULL;
|
|
t->t_forwprev = (t_transition *) NULL;
|
|
t->t_forwnext = (t_transition *) NULL;
|
|
t->t_from = (t_state *) NULL;
|
|
t->t_to = (t_state *) NULL;
|
|
}
|
|
|
|
|
|
void
|
|
link_transition(t_transition * t, t_state * from, t_state * to)
|
|
|
|
{
|
|
if (!from)
|
|
from = t->t_from;
|
|
|
|
if (!to)
|
|
to = t->t_to;
|
|
|
|
unlink_transition(t);
|
|
|
|
if ((t->t_from = from)) {
|
|
if ((t->t_forwnext = from->s_forward))
|
|
t->t_forwnext->t_forwprev = t;
|
|
|
|
from->s_forward = t;
|
|
}
|
|
|
|
if ((t->t_to = to)) {
|
|
if ((t->t_backnext = to->s_backward))
|
|
t->t_backnext->t_backprev = t;
|
|
|
|
to->s_backward = t;
|
|
}
|
|
}
|
|
|
|
|
|
t_transition *
|
|
newtransition(unsigned int token, t_state * from, t_state * to)
|
|
|
|
{
|
|
t_transition * t;
|
|
|
|
t = (t_transition *) malloc(sizeof *t);
|
|
chknull(t);
|
|
memset((char *) t, 0, sizeof *t);
|
|
t->t_token = token;
|
|
link_transition(t, from, to);
|
|
return t;
|
|
}
|
|
|
|
|
|
t_transition *
|
|
uniquetransition(unsigned int token, t_state * from, t_state * to)
|
|
|
|
{
|
|
t_transition * t;
|
|
|
|
for (t = from->s_forward; t; t = t->t_forwnext)
|
|
if (t->t_token == token && (t->t_to == to || !to))
|
|
return t;
|
|
|
|
return to? newtransition(token, from, to): (t_transition *) NULL;
|
|
}
|
|
|
|
|
|
int
|
|
set_position(t_powerset * s, void * e)
|
|
|
|
{
|
|
unsigned int l;
|
|
unsigned int h;
|
|
unsigned int m;
|
|
int i;
|
|
|
|
l = 0;
|
|
h = s->p_card;
|
|
|
|
while (l < h) {
|
|
m = (l + h) >> 1;
|
|
|
|
/**
|
|
*** If both pointers belong to different allocation arenas,
|
|
*** native comparison may find them neither
|
|
*** equal, nor greater, nor smaller.
|
|
*** We thus compare using memcmp() to get an orthogonal
|
|
*** result.
|
|
**/
|
|
|
|
i = memcmp(&e, s->p_set + m, sizeof e);
|
|
|
|
if (i < 0)
|
|
h = m;
|
|
else if (!i)
|
|
return m;
|
|
else
|
|
l = m + 1;
|
|
}
|
|
|
|
return l;
|
|
}
|
|
|
|
|
|
t_powerset *
|
|
set_include(t_powerset * s, void * e)
|
|
|
|
{
|
|
unsigned int pos;
|
|
unsigned int n;
|
|
|
|
if (!s) {
|
|
s = (t_powerset *) malloc(sizeof *s +
|
|
GRANULE * sizeof s->p_set);
|
|
chknull(s);
|
|
s->p_size = GRANULE;
|
|
s->p_set[GRANULE] = (t_state *) NULL;
|
|
s->p_set[0] = e;
|
|
s->p_card = 1;
|
|
return s;
|
|
}
|
|
|
|
pos = set_position(s, e);
|
|
|
|
if (pos < s->p_card && s->p_set[pos] == e)
|
|
return s;
|
|
|
|
if (s->p_card >= s->p_size) {
|
|
s->p_size += GRANULE;
|
|
s = (t_powerset *) realloc(s,
|
|
sizeof *s + s->p_size * sizeof s->p_set);
|
|
chknull(s);
|
|
s->p_set[s->p_size] = (t_state *) NULL;
|
|
}
|
|
|
|
n = s->p_card - pos;
|
|
|
|
if (n)
|
|
memmove((char *) (s->p_set + pos + 1),
|
|
(char *) (s->p_set + pos), n * sizeof s->p_set[0]);
|
|
|
|
s->p_set[pos] = e;
|
|
s->p_card++;
|
|
return s;
|
|
}
|
|
|
|
|
|
t_state *
|
|
nfatransition(t_state * to, byte token)
|
|
|
|
{
|
|
t_state * from;
|
|
|
|
from = newstate();
|
|
newtransition(token, from, to);
|
|
return from;
|
|
}
|
|
|
|
|
|
static t_state * nfadevelop(t_state * from, t_state * final, iconv_t icc,
|
|
const utf8char * name, unsigned int len);
|
|
|
|
|
|
void
|
|
nfaslice(t_state * * from, t_state * * to, iconv_t icc,
|
|
const utf8char * chr, unsigned int chlen,
|
|
const utf8char * name, unsigned int len, t_state * final)
|
|
|
|
{
|
|
char * srcp;
|
|
char * dstp;
|
|
size_t srcc;
|
|
size_t dstc;
|
|
unsigned int cnt;
|
|
t_state * f;
|
|
t_state * t;
|
|
t_transition * tp;
|
|
byte bytebuf[8];
|
|
|
|
srcp = (char *) chr;
|
|
srcc = chlen;
|
|
dstp = (char *) bytebuf;
|
|
dstc = sizeof bytebuf;
|
|
iconv(icc, &srcp, &srcc, &dstp, &dstc);
|
|
dstp = (char *) bytebuf;
|
|
cnt = sizeof bytebuf - dstc;
|
|
t = *to;
|
|
f = *from;
|
|
|
|
/**
|
|
*** Check for end of string.
|
|
**/
|
|
|
|
if (!len)
|
|
if (t && t != final)
|
|
uniquetransition(EPSILON, t, final);
|
|
else
|
|
t = final;
|
|
|
|
if (f)
|
|
while (cnt) {
|
|
tp = uniquetransition(*dstp, f, (t_state *) NULL);
|
|
|
|
if (!tp)
|
|
break;
|
|
|
|
f = tp->t_to;
|
|
dstp++;
|
|
cnt--;
|
|
}
|
|
|
|
if (!cnt) {
|
|
if (!t)
|
|
t = nfadevelop(f, final, icc, name, len);
|
|
|
|
*to = t;
|
|
return;
|
|
}
|
|
|
|
if (!t) {
|
|
t = nfadevelop((t_state *) NULL, final, icc, name, len);
|
|
*to = t;
|
|
}
|
|
|
|
if (!f)
|
|
*from = f = newstate();
|
|
|
|
while (cnt > 1)
|
|
t = nfatransition(t, dstp[--cnt]);
|
|
|
|
newtransition(*dstp, f, t);
|
|
}
|
|
|
|
|
|
t_state *
|
|
nfadevelop(t_state * from, t_state * final, iconv_t icc,
|
|
const utf8char * name, unsigned int len)
|
|
|
|
{
|
|
int chlen;
|
|
int i;
|
|
t_state * to;
|
|
int uccnt;
|
|
int lccnt;
|
|
utf8char chr;
|
|
|
|
chlen = utf8_chlen[*name];
|
|
|
|
for (i = 1; i < chlen; i++)
|
|
if ((name[i] & 0xC0) != 0x80)
|
|
break;
|
|
|
|
if (i != chlen) {
|
|
fprintf(stderr,
|
|
"Invalid UTF8 character in character set name\n");
|
|
return (t_state *) NULL;
|
|
}
|
|
|
|
to = (t_state *) NULL;
|
|
nfaslice(&from, &to,
|
|
icc, name, chlen, name + chlen, len - chlen, final);
|
|
|
|
if (*name >= UTF8_a && *name <= UTF8_z)
|
|
chr = *name - UTF8_a + UTF8_A;
|
|
else if (*name >= UTF8_A && *name <= UTF8_Z)
|
|
chr = *name - UTF8_A + UTF8_a;
|
|
else
|
|
return from;
|
|
|
|
nfaslice(&from, &to, icc, &chr, 1, name + chlen, len - chlen, final);
|
|
return from;
|
|
}
|
|
|
|
|
|
|
|
void
|
|
nfaenter(const utf8char * name, int len, t_chset * charset)
|
|
|
|
{
|
|
t_chset * s;
|
|
t_state * final;
|
|
t_state * sp;
|
|
t_symlist * lp;
|
|
|
|
/**
|
|
*** Enter case-insensitive `name' in NFA in all known
|
|
*** character codes.
|
|
*** Redundant shift state changes as well as shift state
|
|
*** differences between uppercase and lowercase are
|
|
*** not handled.
|
|
**/
|
|
|
|
if (len < 0)
|
|
len = strlen(name) + 1;
|
|
|
|
for (lp = charset->c_names; lp; lp = lp->l_next)
|
|
if (!memcmp(name, lp->l_symbol, len))
|
|
return; /* Already entered. */
|
|
|
|
lp = (t_symlist *) malloc(sizeof *lp + len);
|
|
chknull(lp);
|
|
memcpy(lp->l_symbol, name, len);
|
|
lp->l_symbol[len] = '\0';
|
|
lp->l_next = charset->c_names;
|
|
charset->c_names = lp;
|
|
final = newstate();
|
|
final->s_final = charset;
|
|
|
|
for (s = chset_list; s; s = s->c_next)
|
|
if (!iconv_open_error(s->c_fromUTF8))
|
|
sp = nfadevelop(initial_state, final,
|
|
s->c_fromUTF8, name, len);
|
|
}
|
|
|
|
|
|
unsigned int
|
|
utf8_utostr(utf8char * s, unsigned int v)
|
|
|
|
{
|
|
unsigned int d;
|
|
unsigned int i;
|
|
|
|
d = v / 10;
|
|
v -= d * 10;
|
|
i = d? utf8_utostr(s, d): 0;
|
|
s[i++] = v + UTF8_0;
|
|
s[i] = '\0';
|
|
return i;
|
|
}
|
|
|
|
|
|
unsigned int
|
|
utf8_utostrpad(utf8char * s, unsigned int v, int digits)
|
|
|
|
{
|
|
unsigned int i = utf8_utostr(s, v);
|
|
utf8char pad = UTF8_SPACE;
|
|
|
|
if (digits < 0) {
|
|
pad = UTF8_0;
|
|
digits = -digits;
|
|
}
|
|
|
|
if (i >= digits)
|
|
return i;
|
|
|
|
memmove(s + digits - i, s, i + 1);
|
|
memset(s, pad, digits - i);
|
|
return digits;
|
|
}
|
|
|
|
|
|
unsigned int
|
|
utf8_strtou(const utf8char * s)
|
|
|
|
{
|
|
unsigned int v;
|
|
|
|
while (*s == UTF8_SPACE || *s == UTF8_HT)
|
|
s++;
|
|
|
|
for (v = 0; *s >= UTF8_0 && *s <= UTF8_9;)
|
|
v = 10 * v + *s++ - UTF8_0;
|
|
|
|
return v;
|
|
}
|
|
|
|
|
|
unsigned int
|
|
getNumAttr(xmlNodePtr node, const xmlChar * name)
|
|
|
|
{
|
|
const xmlChar * s;
|
|
unsigned int val;
|
|
|
|
s = xmlGetProp(node, name);
|
|
|
|
if (!s)
|
|
return 0;
|
|
|
|
val = utf8_strtou(s);
|
|
xmlFree((xmlChar *) s);
|
|
return val;
|
|
}
|
|
|
|
|
|
void
|
|
read_assocs(const char * filename)
|
|
|
|
{
|
|
xmlDocPtr doc;
|
|
xmlXPathContextPtr ctxt;
|
|
xmlXPathObjectPtr obj;
|
|
xmlNodePtr node;
|
|
t_chset * sp;
|
|
int i;
|
|
unsigned int ccsid;
|
|
unsigned int mibenum;
|
|
utf8char symbuf[32];
|
|
|
|
doc = loadXMLFile(filename);
|
|
|
|
if (!doc) {
|
|
fprintf(stderr, "Cannot load file %s\n", filename);
|
|
exit(1);
|
|
}
|
|
|
|
ctxt = xmlXPathNewContext(doc);
|
|
obj = xmlXPathEval(utf8_assocnodes, ctxt);
|
|
|
|
if (!obj || obj->type != XPATH_NODESET || !obj->nodesetval ||
|
|
!obj->nodesetval->nodeTab || !obj->nodesetval->nodeNr) {
|
|
fprintf(stderr, "No association found in %s\n", filename);
|
|
exit(1);
|
|
}
|
|
|
|
for (i = 0; i < obj->nodesetval->nodeNr; i++) {
|
|
node = obj->nodesetval->nodeTab[i];
|
|
ccsid = getNumAttr(node, utf8_ccsid);
|
|
mibenum = getNumAttr(node, utf8_mibenum);
|
|
|
|
/**
|
|
*** Check for duplicate.
|
|
**/
|
|
|
|
for (sp = chset_list; sp; sp = sp->c_next)
|
|
if (ccsid && ccsid == sp->c_ccsid ||
|
|
mibenum && mibenum == sp->c_mibenum) {
|
|
fprintf(stderr, "Duplicate character set: ");
|
|
fprintf(stderr, "CCSID = %u/%u, ",
|
|
ccsid, sp->c_ccsid);
|
|
fprintf(stderr, "MIBenum = %u/%u\n",
|
|
mibenum, sp->c_mibenum);
|
|
break;
|
|
}
|
|
|
|
if (sp)
|
|
continue;
|
|
|
|
/**
|
|
*** Allocate the new character set.
|
|
**/
|
|
|
|
sp = (t_chset *) malloc(sizeof *sp);
|
|
chknull(sp);
|
|
memset(sp, 0, sizeof *sp);
|
|
|
|
if (!ccsid) /* Do not attempt with current job CCSID. */
|
|
set_iconv_open_error(sp->c_fromUTF8);
|
|
else {
|
|
sp->c_fromUTF8 =
|
|
iconv_open_ccsid(ccsid, C_UTF8_CCSID, 0);
|
|
|
|
if (iconv_open_error(sp->c_fromUTF8) == -1)
|
|
fprintf(stderr,
|
|
"Cannot convert into CCSID %u: ignored\n",
|
|
ccsid);
|
|
}
|
|
|
|
sp->c_ccsid = ccsid;
|
|
sp->c_mibenum = mibenum;
|
|
sp->c_next = chset_list;
|
|
chset_list = sp;
|
|
}
|
|
|
|
xmlXPathFreeObject(obj);
|
|
|
|
/**
|
|
*** Enter aliases.
|
|
**/
|
|
|
|
for (sp = chset_list; sp; sp = sp->c_next) {
|
|
strcpy(symbuf, utf8_ibm_);
|
|
utf8_utostr(symbuf + 4, sp->c_ccsid);
|
|
nfaenter(symbuf, -1, sp);
|
|
strcpy(symbuf, utf8_IBMCCSID);
|
|
utf8_utostrpad(symbuf + 8, sp->c_ccsid, -5);
|
|
nfaenter(symbuf, 13, sp); /* Not null-terminated. */
|
|
|
|
if (sp->c_mibenum) {
|
|
strcpy(symbuf, utf8_iana_);
|
|
utf8_utostr(symbuf + 5, sp->c_mibenum);
|
|
nfaenter(symbuf, -1, sp);
|
|
}
|
|
|
|
xmlXPathRegisterVariable(ctxt, utf8_C,
|
|
xmlXPathNewFloat((double) sp->c_ccsid));
|
|
obj = xmlXPathEval(utf8_aliastext, ctxt);
|
|
|
|
if (!obj || obj->type != XPATH_NODESET) {
|
|
fprintf(stderr, "getAlias failed in %s\n", filename);
|
|
exit(1);
|
|
}
|
|
|
|
if (obj->nodesetval &&
|
|
obj->nodesetval->nodeTab && obj->nodesetval->nodeNr) {
|
|
for (i = 0; i < obj->nodesetval->nodeNr; i++) {
|
|
node = obj->nodesetval->nodeTab[i];
|
|
nfaenter(node->content, -1, sp);
|
|
}
|
|
}
|
|
|
|
xmlXPathFreeObject(obj);
|
|
}
|
|
|
|
xmlXPathFreeContext(ctxt);
|
|
xmlFreeDoc(doc);
|
|
}
|
|
|
|
|
|
unsigned int
|
|
columnPosition(xmlXPathContextPtr ctxt, const xmlChar * header)
|
|
|
|
{
|
|
xmlXPathObjectPtr obj;
|
|
unsigned int res = 0;
|
|
|
|
xmlXPathRegisterVariable(ctxt, utf8_T, xmlXPathNewString(header));
|
|
obj = xmlXPathEval(utf8_headerpos, ctxt);
|
|
|
|
if (obj) {
|
|
if (obj->type == XPATH_NUMBER)
|
|
res = (unsigned int) obj->floatval;
|
|
|
|
xmlXPathFreeObject(obj);
|
|
}
|
|
|
|
return res;
|
|
}
|
|
|
|
|
|
void
|
|
read_iana(const char * filename)
|
|
|
|
{
|
|
xmlDocPtr doc;
|
|
xmlXPathContextPtr ctxt;
|
|
xmlXPathObjectPtr obj1;
|
|
xmlXPathObjectPtr obj2;
|
|
xmlNodePtr node;
|
|
int prefnamecol;
|
|
int namecol;
|
|
int mibenumcol;
|
|
int aliascol;
|
|
int mibenum;
|
|
t_chset * sp;
|
|
int n;
|
|
int i;
|
|
|
|
doc = loadXMLFile(filename);
|
|
|
|
if (!doc) {
|
|
fprintf(stderr, "Cannot load file %s\n", filename);
|
|
exit(1);
|
|
}
|
|
|
|
ctxt = xmlXPathNewContext(doc);
|
|
|
|
#ifndef OLDXML
|
|
xmlXPathRegisterNs(ctxt, utf8_html, utf8_htmluri);
|
|
#endif
|
|
|
|
obj1 = xmlXPathEval(utf8_tablerows, ctxt);
|
|
|
|
if (!obj1 || obj1->type != XPATH_NODESET || !obj1->nodesetval ||
|
|
!obj1->nodesetval->nodeTab || obj1->nodesetval->nodeNr <= 1) {
|
|
fprintf(stderr, "No data in %s\n", filename);
|
|
exit(1);
|
|
}
|
|
|
|
/**
|
|
*** Identify columns.
|
|
**/
|
|
|
|
xmlXPathSetContextNode(obj1->nodesetval->nodeTab[0], ctxt);
|
|
prefnamecol = columnPosition(ctxt, utf8_Pref_MIME_Name);
|
|
namecol = columnPosition(ctxt, utf8_Name);
|
|
mibenumcol = columnPosition(ctxt, utf8_MIBenum);
|
|
aliascol = columnPosition(ctxt, utf8_Aliases);
|
|
|
|
if (!prefnamecol || !namecol || !mibenumcol || !aliascol) {
|
|
fprintf(stderr, "Key column(s) missing in %s\n", filename);
|
|
exit(1);
|
|
}
|
|
|
|
xmlXPathRegisterVariable(ctxt, utf8_P,
|
|
xmlXPathNewFloat((double) prefnamecol));
|
|
xmlXPathRegisterVariable(ctxt, utf8_N,
|
|
xmlXPathNewFloat((double) namecol));
|
|
xmlXPathRegisterVariable(ctxt, utf8_M,
|
|
xmlXPathNewFloat((double) mibenumcol));
|
|
xmlXPathRegisterVariable(ctxt, utf8_A,
|
|
xmlXPathNewFloat((double) aliascol));
|
|
|
|
/**
|
|
*** Process each row.
|
|
**/
|
|
|
|
for (n = 1; n < obj1->nodesetval->nodeNr; n++) {
|
|
xmlXPathSetContextNode(obj1->nodesetval->nodeTab[n], ctxt);
|
|
|
|
/**
|
|
*** Get the MIBenum from current row.
|
|
*/
|
|
|
|
obj2 = xmlXPathEval(utf8_getmibenum, ctxt);
|
|
|
|
if (!obj2 || obj2->type != XPATH_NUMBER) {
|
|
fprintf(stderr, "get MIBenum failed at row %u\n", n);
|
|
exit(1);
|
|
}
|
|
|
|
if (xmlXPathIsNaN(obj2->floatval) ||
|
|
obj2->floatval < 1.0 || obj2->floatval > 65535.0 ||
|
|
((unsigned int) obj2->floatval) != obj2->floatval) {
|
|
fprintf(stderr, "invalid MIBenum at row %u\n", n);
|
|
xmlXPathFreeObject(obj2);
|
|
continue;
|
|
}
|
|
|
|
mibenum = obj2->floatval;
|
|
xmlXPathFreeObject(obj2);
|
|
|
|
/**
|
|
*** Search the associations for a corresponding CCSID.
|
|
**/
|
|
|
|
for (sp = chset_list; sp; sp = sp->c_next)
|
|
if (sp->c_mibenum == mibenum)
|
|
break;
|
|
|
|
if (!sp)
|
|
continue; /* No CCSID for this MIBenum. */
|
|
|
|
/**
|
|
*** Process preferred MIME name.
|
|
**/
|
|
|
|
obj2 = xmlXPathEval(utf8_getprefname, ctxt);
|
|
|
|
if (!obj2 || obj2->type != XPATH_STRING) {
|
|
fprintf(stderr,
|
|
"get Preferred_MIME_Name failed at row %u\n", n);
|
|
exit(1);
|
|
}
|
|
|
|
if (obj2->stringval && obj2->stringval[0])
|
|
nfaenter(obj2->stringval, -1, sp);
|
|
|
|
xmlXPathFreeObject(obj2);
|
|
|
|
/**
|
|
*** Process name.
|
|
**/
|
|
|
|
obj2 = xmlXPathEval(utf8_getname, ctxt);
|
|
|
|
if (!obj2 || obj2->type != XPATH_STRING) {
|
|
fprintf(stderr, "get name failed at row %u\n", n);
|
|
exit(1);
|
|
}
|
|
|
|
if (obj2->stringval && obj2->stringval[0])
|
|
nfaenter(obj2->stringval, -1, sp);
|
|
|
|
xmlXPathFreeObject(obj2);
|
|
|
|
/**
|
|
*** Process aliases.
|
|
**/
|
|
|
|
obj2 = xmlXPathEval(utf8_getaliases, ctxt);
|
|
|
|
if (!obj2 || obj2->type != XPATH_NODESET) {
|
|
fprintf(stderr, "get aliases failed at row %u\n", n);
|
|
exit(1);
|
|
}
|
|
|
|
if (obj2->nodesetval && obj2->nodesetval->nodeTab)
|
|
for (i = 0; i < obj2->nodesetval->nodeNr; i++) {
|
|
node = obj2->nodesetval->nodeTab[i];
|
|
|
|
if (node && node->content && node->content[0])
|
|
nfaenter(node->content, -1, sp);
|
|
}
|
|
|
|
xmlXPathFreeObject(obj2);
|
|
}
|
|
|
|
xmlXPathFreeObject(obj1);
|
|
xmlXPathFreeContext(ctxt);
|
|
xmlFreeDoc(doc);
|
|
}
|
|
|
|
|
|
t_powerset * closureset(t_powerset * dst, t_powerset * src);
|
|
|
|
|
|
t_powerset *
|
|
closure(t_powerset * dst, t_state * src)
|
|
|
|
{
|
|
t_transition * t;
|
|
unsigned int oldcard;
|
|
|
|
if (src->s_nfastates) {
|
|
/**
|
|
*** Is a DFA state: return closure of set of equivalent
|
|
*** NFA states.
|
|
**/
|
|
|
|
return closureset(dst, src->s_nfastates);
|
|
}
|
|
|
|
/**
|
|
*** Compute closure of NFA state.
|
|
**/
|
|
|
|
dst = set_include(dst, src);
|
|
|
|
for (t = src->s_forward; t; t = t->t_forwnext)
|
|
if (t->t_token == EPSILON) {
|
|
oldcard = dst->p_card;
|
|
dst = set_include(dst, t->t_to);
|
|
|
|
if (oldcard != dst->p_card)
|
|
dst = closure(dst, t->t_to);
|
|
}
|
|
|
|
return dst;
|
|
}
|
|
|
|
|
|
t_powerset *
|
|
closureset(t_powerset * dst, t_powerset * src)
|
|
|
|
{
|
|
unsigned int i;
|
|
|
|
for (i = 0; i < src->p_card; i++)
|
|
dst = closure(dst, (t_state *) src->p_set[i]);
|
|
|
|
return dst;
|
|
}
|
|
|
|
|
|
t_state *
|
|
get_dfa_state(t_state * * stack,
|
|
t_powerset * nfastates, xmlHashTablePtr sethash)
|
|
|
|
{
|
|
t_state * s;
|
|
|
|
if (s = hash_get(sethash, nfastates->p_set,
|
|
nfastates->p_card * sizeof nfastates->p_set[0])) {
|
|
/**
|
|
*** DFA state already present.
|
|
*** Release the NFA state set and return
|
|
*** the address of the old DFA state.
|
|
**/
|
|
|
|
free((char *) nfastates);
|
|
return s;
|
|
}
|
|
|
|
/**
|
|
*** Build the new state.
|
|
**/
|
|
|
|
s = newstate();
|
|
s->s_nfastates = nfastates;
|
|
s->s_next = dfa_states;
|
|
dfa_states = s;
|
|
s->s_stack = *stack;
|
|
*stack = s;
|
|
|
|
/**
|
|
*** Enter it in hash.
|
|
**/
|
|
|
|
if (hash_add(sethash, nfastates->p_set,
|
|
nfastates->p_card * sizeof nfastates->p_set[0], s))
|
|
chknull(NULL); /* Memory allocation error. */
|
|
|
|
return s;
|
|
}
|
|
|
|
|
|
int
|
|
transcmp(const void * p1, const void * p2)
|
|
|
|
{
|
|
t_transition * t1;
|
|
t_transition * t2;
|
|
|
|
t1 = *(t_transition * *) p1;
|
|
t2 = *(t_transition * *) p2;
|
|
return ((int) t1->t_token) - ((int) t2->t_token);
|
|
}
|
|
|
|
|
|
void
|
|
builddfa(void)
|
|
|
|
{
|
|
t_powerset * transset;
|
|
t_powerset * stateset;
|
|
t_state * s;
|
|
t_state * s2;
|
|
unsigned int n;
|
|
unsigned int i;
|
|
unsigned int token;
|
|
t_transition * t;
|
|
t_state * stack;
|
|
xmlHashTablePtr sethash;
|
|
unsigned int nst;
|
|
|
|
transset = set_include(NULL, NULL);
|
|
chknull(transset);
|
|
stateset = set_include(NULL, NULL);
|
|
chknull(stateset);
|
|
sethash = xmlHashCreate(1);
|
|
chknull(sethash);
|
|
dfa_states = (t_state *) NULL;
|
|
stack = (t_state *) NULL;
|
|
nst = 0;
|
|
|
|
/**
|
|
*** Build the DFA initial state.
|
|
**/
|
|
|
|
get_dfa_state(&stack, closure(NULL, initial_state), sethash);
|
|
|
|
/**
|
|
*** Build the other DFA states by looking at each
|
|
*** possible transition from stacked DFA states.
|
|
**/
|
|
|
|
do {
|
|
if (!(++nst % 100))
|
|
fprintf(stderr, "%u DFA states\n", nst);
|
|
|
|
s = stack;
|
|
stack = s->s_stack;
|
|
s->s_stack = (t_state *) NULL;
|
|
|
|
/**
|
|
*** Build a set of all non-epsilon transitions from this
|
|
*** state.
|
|
**/
|
|
|
|
transset->p_card = 0;
|
|
|
|
for (n = 0; n < s->s_nfastates->p_card; n++) {
|
|
s2 = s->s_nfastates->p_set[n];
|
|
|
|
for (t = s2->s_forward; t; t = t->t_forwnext)
|
|
if (t->t_token != EPSILON) {
|
|
transset = set_include(transset, t);
|
|
chknull(transset);
|
|
}
|
|
}
|
|
|
|
/**
|
|
*** Sort transitions by token.
|
|
**/
|
|
|
|
qsort(transset->p_set, transset->p_card,
|
|
sizeof transset->p_set[0], transcmp);
|
|
|
|
/**
|
|
*** Process all transitions, grouping them by token.
|
|
**/
|
|
|
|
stateset->p_card = 0;
|
|
token = EPSILON;
|
|
|
|
for (i = 0; i < transset->p_card; i++) {
|
|
t = transset->p_set[i];
|
|
|
|
if (token != t->t_token) {
|
|
if (stateset->p_card) {
|
|
/**
|
|
*** Get the equivalent DFA state
|
|
*** and create transition.
|
|
**/
|
|
|
|
newtransition(token, s,
|
|
get_dfa_state(&stack,
|
|
closureset(NULL, stateset),
|
|
sethash));
|
|
stateset->p_card = 0;
|
|
}
|
|
|
|
token = t->t_token;
|
|
}
|
|
|
|
stateset = set_include(stateset, t->t_to);
|
|
}
|
|
|
|
if (stateset->p_card)
|
|
newtransition(token, s, get_dfa_state(&stack,
|
|
closureset(NULL, stateset), sethash));
|
|
} while (stack);
|
|
|
|
free((char *) transset);
|
|
free((char *) stateset);
|
|
xmlHashFree(sethash, NULL);
|
|
|
|
/**
|
|
*** Reverse the state list to get the initial state first,
|
|
*** check for ambiguous prefixes, determine final states,
|
|
*** destroy NFA state sets.
|
|
**/
|
|
|
|
while (s = dfa_states) {
|
|
dfa_states = s->s_next;
|
|
s->s_next = stack;
|
|
stack = s;
|
|
stateset = s->s_nfastates;
|
|
s->s_nfastates = (t_powerset *) NULL;
|
|
|
|
for (n = 0; n < stateset->p_card; n++) {
|
|
s2 = (t_state *) stateset->p_set[n];
|
|
|
|
if (s2->s_final) {
|
|
if (s->s_final && s->s_final != s2->s_final)
|
|
fprintf(stderr,
|
|
"Ambiguous name for CCSIDs %u/%u\n",
|
|
s->s_final->c_ccsid,
|
|
s2->s_final->c_ccsid);
|
|
|
|
s->s_final = s2->s_final;
|
|
}
|
|
}
|
|
|
|
free((char *) stateset);
|
|
}
|
|
|
|
dfa_states = stack;
|
|
}
|
|
|
|
|
|
void
|
|
deletenfa(void)
|
|
|
|
{
|
|
t_transition * t;
|
|
t_state * s;
|
|
t_state * u;
|
|
t_state * stack;
|
|
|
|
stack = initial_state;
|
|
stack->s_stack = (t_state *) NULL;
|
|
|
|
while ((s = stack)) {
|
|
stack = s->s_stack;
|
|
|
|
while ((t = s->s_forward)) {
|
|
u = t->t_to;
|
|
unlink_transition(t);
|
|
free((char *) t);
|
|
|
|
if (!u->s_backward) {
|
|
u->s_stack = stack;
|
|
stack = u;
|
|
}
|
|
}
|
|
|
|
free((char *) s);
|
|
}
|
|
}
|
|
|
|
|
|
t_stategroup *
|
|
newgroup(void)
|
|
|
|
{
|
|
t_stategroup * g;
|
|
|
|
g = (t_stategroup *) malloc(sizeof *g);
|
|
chknull(g);
|
|
memset((char *) g, 0, sizeof *g);
|
|
g->g_id = groupid++;
|
|
return g;
|
|
}
|
|
|
|
|
|
void
|
|
optimizedfa(void)
|
|
|
|
{
|
|
unsigned int i;
|
|
xmlHashTablePtr h;
|
|
t_state * s1;
|
|
t_state * s2;
|
|
t_state * finstates;
|
|
t_state * * sp;
|
|
t_stategroup * g1;
|
|
t_stategroup * g2;
|
|
t_stategroup * ghead;
|
|
t_transition * t1;
|
|
t_transition * t2;
|
|
unsigned int done;
|
|
unsigned int startgroup;
|
|
unsigned int gtrans[1 << (8 * sizeof(unsigned char))];
|
|
|
|
/**
|
|
*** Reduce DFA state count.
|
|
**/
|
|
|
|
groupid = 0;
|
|
ghead = (t_stategroup *) NULL;
|
|
|
|
/**
|
|
*** First split: non-final and each distinct final states.
|
|
**/
|
|
|
|
h = xmlHashCreate(4);
|
|
chknull(h);
|
|
|
|
for (s1 = dfa_states; s1; s1 = s1->s_next) {
|
|
if (!(g1 = hash_get(h, &s1->s_final, sizeof s1->s_final))) {
|
|
g1 = newgroup();
|
|
g1->g_next = ghead;
|
|
ghead = g1;
|
|
|
|
if (hash_add(h, &s1->s_final, sizeof s1->s_final, g1))
|
|
chknull(NULL); /* Memory allocation error. */
|
|
}
|
|
|
|
s1->s_index = g1->g_id;
|
|
s1->s_stack = g1->g_member;
|
|
g1->g_member = s1;
|
|
}
|
|
|
|
xmlHashFree(h, NULL);
|
|
|
|
/**
|
|
*** Subsequent splits: states that have the same forward
|
|
*** transition tokens to states in the same group.
|
|
**/
|
|
|
|
do {
|
|
done = 1;
|
|
|
|
for (g2 = ghead; g2; g2 = g2->g_next) {
|
|
s1 = g2->g_member;
|
|
|
|
if (!s1->s_stack)
|
|
continue;
|
|
|
|
h = xmlHashCreate(1);
|
|
chknull(h);
|
|
|
|
/**
|
|
*** Build the group transition map.
|
|
**/
|
|
|
|
memset((char *) gtrans, ~0, sizeof gtrans);
|
|
|
|
for (t1 = s1->s_forward; t1; t1 = t1->t_forwnext)
|
|
gtrans[t1->t_token] = t1->t_to->s_index;
|
|
|
|
if (hash_add(h, gtrans, sizeof gtrans, g2))
|
|
chknull(NULL);
|
|
|
|
/**
|
|
*** Process other states in group.
|
|
**/
|
|
|
|
sp = &s1->s_stack;
|
|
s1 = *sp;
|
|
|
|
do {
|
|
*sp = s1->s_stack;
|
|
|
|
/**
|
|
*** Build the transition map.
|
|
**/
|
|
|
|
memset((char *) gtrans, ~0, sizeof gtrans);
|
|
|
|
for (t1 = s1->s_forward;
|
|
t1; t1 = t1->t_forwnext)
|
|
gtrans[t1->t_token] = t1->t_to->s_index;
|
|
|
|
g1 = hash_get(h, gtrans, sizeof gtrans);
|
|
|
|
if (g1 == g2) {
|
|
*sp = s1;
|
|
sp = &s1->s_stack;
|
|
}
|
|
else {
|
|
if (!g1) {
|
|
g1 = newgroup();
|
|
g1->g_next = ghead;
|
|
ghead = g1;
|
|
|
|
if (hash_add(h, gtrans,
|
|
sizeof gtrans, g1))
|
|
chknull(NULL);
|
|
}
|
|
|
|
s1->s_index = g1->g_id;
|
|
s1->s_stack = g1->g_member;
|
|
g1->g_member = s1;
|
|
done = 0;
|
|
}
|
|
} while (s1 = *sp);
|
|
|
|
xmlHashFree(h, NULL);
|
|
}
|
|
} while (!done);
|
|
|
|
/**
|
|
*** Establish group leaders and remap transitions.
|
|
**/
|
|
|
|
startgroup = dfa_states->s_index;
|
|
|
|
for (g1 = ghead; g1; g1 = g1->g_next)
|
|
for (s1 = g1->g_member->s_stack; s1; s1 = s1->s_stack)
|
|
for (t1 = s1->s_backward; t1; t1 = t2) {
|
|
t2 = t1->t_backnext;
|
|
link_transition(t1, NULL, g1->g_member);
|
|
}
|
|
|
|
/**
|
|
*** Remove redundant states and transitions.
|
|
**/
|
|
|
|
for (g1 = ghead; g1; g1 = g1->g_next) {
|
|
g1->g_member->s_next = (t_state *) NULL;
|
|
|
|
while ((s1 = g1->g_member->s_stack)) {
|
|
g1->g_member->s_stack = s1->s_stack;
|
|
|
|
for (t1 = s1->s_forward; t1; t1 = t2) {
|
|
t2 = t1->t_forwnext;
|
|
unlink_transition(t1);
|
|
free((char *) t1);
|
|
}
|
|
|
|
free((char *) s1);
|
|
}
|
|
}
|
|
|
|
/**
|
|
*** Remove group support and relink DFA states.
|
|
**/
|
|
|
|
dfa_states = (t_state *) NULL;
|
|
s2 = (t_state *) NULL;
|
|
finstates = (t_state *) NULL;
|
|
|
|
while (g1 = ghead) {
|
|
ghead = g1->g_next;
|
|
s1 = g1->g_member;
|
|
|
|
if (g1->g_id == startgroup)
|
|
dfa_states = s1; /* Keep start state first. */
|
|
else if (s1->s_final) { /* Then final states. */
|
|
s1->s_next = finstates;
|
|
finstates = s1;
|
|
}
|
|
else { /* Finish with non-final states. */
|
|
s1->s_next = s2;
|
|
s2 = s1;
|
|
}
|
|
|
|
free((char *) g1);
|
|
}
|
|
|
|
for (dfa_states->s_next = finstates; finstates->s_next;)
|
|
finstates = finstates->s_next;
|
|
|
|
finstates->s_next = s2;
|
|
}
|
|
|
|
|
|
const char *
|
|
inttype(unsigned long max)
|
|
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; max; i++)
|
|
max >>= 1;
|
|
|
|
if (i > 8 * sizeof(unsigned int))
|
|
return "unsigned long";
|
|
|
|
if (i > 8 * sizeof(unsigned short))
|
|
return "unsigned int";
|
|
|
|
if (i > 8 * sizeof(unsigned char))
|
|
return "unsigned short";
|
|
|
|
return "unsigned char";
|
|
}
|
|
|
|
|
|
listids(FILE * fp)
|
|
|
|
{
|
|
unsigned int pos;
|
|
t_chset * cp;
|
|
t_symlist * lp;
|
|
char * srcp;
|
|
char * dstp;
|
|
size_t srcc;
|
|
size_t dstc;
|
|
char buf[80];
|
|
|
|
fprintf(fp, "/**\n*** CCSID For arg Recognized name.\n");
|
|
pos = 0;
|
|
|
|
for (cp = chset_list; cp; cp = cp->c_next) {
|
|
if (pos) {
|
|
fprintf(fp, "\n");
|
|
pos = 0;
|
|
}
|
|
|
|
if (!cp->c_names)
|
|
continue;
|
|
|
|
pos = fprintf(fp, "*** %5u %c ", cp->c_ccsid,
|
|
iconv_open_error(cp->c_fromUTF8)? ' ': 'X');
|
|
|
|
for (lp = cp->c_names; lp; lp = lp->l_next) {
|
|
srcp = (char *) lp->l_symbol;
|
|
srcc = strlen(srcp);
|
|
dstp = buf;
|
|
dstc = sizeof buf;
|
|
iconv(utf82job, &srcp, &srcc, &dstp, &dstc);
|
|
srcc = dstp - buf;
|
|
|
|
if (pos + srcc > 79) {
|
|
fprintf(fp, "\n***%22c", ' ');
|
|
pos = 25;
|
|
}
|
|
|
|
pos += fprintf(fp, " %.*s", srcc, buf);
|
|
}
|
|
}
|
|
|
|
if (pos)
|
|
fprintf(fp, "\n");
|
|
|
|
fprintf(fp, "**/\n\n");
|
|
}
|
|
|
|
|
|
void
|
|
generate(FILE * fp)
|
|
|
|
{
|
|
unsigned int nstates;
|
|
unsigned int ntrans;
|
|
unsigned int maxfinal;
|
|
t_state * s;
|
|
t_transition * t;
|
|
unsigned int i;
|
|
unsigned int pos;
|
|
char * ns;
|
|
|
|
/**
|
|
*** Assign indexes to states and transitions.
|
|
**/
|
|
|
|
nstates = 0;
|
|
ntrans = 0;
|
|
maxfinal = 0;
|
|
|
|
for (s = dfa_states; s; s = s->s_next) {
|
|
s->s_index = nstates++;
|
|
|
|
if (s->s_final)
|
|
maxfinal = nstates;
|
|
|
|
for (t = s->s_forward; t; t = t->t_forwnext)
|
|
t->t_index = ntrans++;
|
|
}
|
|
|
|
fprintf(fp,
|
|
"/**\n*** %u states, %u finals, %u transitions.\n**/\n\n",
|
|
nstates, maxfinal, ntrans);
|
|
fprintf(stderr, "%u states, %u finals, %u transitions.\n",
|
|
nstates, maxfinal, ntrans);
|
|
|
|
/**
|
|
*** Generate types.
|
|
**/
|
|
|
|
fprintf(fp, "typedef unsigned short t_ccsid;\n");
|
|
fprintf(fp, "typedef %-23s t_staterange;\n", inttype(nstates));
|
|
fprintf(fp, "typedef %-23s t_transrange;\n\n", inttype(ntrans));
|
|
|
|
/**
|
|
*** Generate first transition index for each state.
|
|
**/
|
|
|
|
fprintf(fp, "static t_transrange trans_array[] = {\n");
|
|
pos = 0;
|
|
ntrans = 0;
|
|
|
|
for (s = dfa_states; s; s = s->s_next) {
|
|
pos += fprintf(fp, " %u,", ntrans);
|
|
|
|
if (pos > 72) {
|
|
fprintf(fp, "\n");
|
|
pos = 0;
|
|
}
|
|
|
|
for (t = s->s_forward; t; t = t->t_forwnext)
|
|
ntrans++;
|
|
}
|
|
|
|
fprintf(fp, " %u\n};\n\n", ntrans);
|
|
|
|
/**
|
|
*** Generate final state info.
|
|
**/
|
|
|
|
fprintf(fp, "static t_ccsid final_array[] = {\n");
|
|
pos = 0;
|
|
ns ="";
|
|
i = 0;
|
|
|
|
for (s = dfa_states; s && i++ < maxfinal; s = s->s_next) {
|
|
pos += fprintf(fp, "%s", ns);
|
|
ns = ",";
|
|
|
|
if (pos > 72) {
|
|
fprintf(fp, "\n");
|
|
pos = 0;
|
|
}
|
|
|
|
pos += fprintf(fp, " %u",
|
|
s->s_final? s->s_final->c_ccsid + 1: 0);
|
|
}
|
|
|
|
fprintf(fp, "\n};\n\n");
|
|
|
|
/**
|
|
*** Generate goto table.
|
|
**/
|
|
|
|
fprintf(fp, "static t_staterange goto_array[] = {\n");
|
|
pos = 0;
|
|
|
|
for (s = dfa_states; s; s = s->s_next)
|
|
for (t = s->s_forward; t; t = t->t_forwnext) {
|
|
pos += fprintf(fp, " %u,", t->t_to->s_index);
|
|
|
|
if (pos > 72) {
|
|
fprintf(fp, "\n");
|
|
pos = 0;
|
|
}
|
|
}
|
|
|
|
fprintf(fp, " %u\n};\n\n", nstates);
|
|
|
|
/**
|
|
*** Generate transition label table.
|
|
**/
|
|
|
|
fprintf(fp, "static unsigned char label_array[] = {\n");
|
|
pos = 0;
|
|
ns ="";
|
|
|
|
for (s = dfa_states; s; s = s->s_next)
|
|
for (t = s->s_forward; t; t = t->t_forwnext) {
|
|
pos += fprintf(fp, "%s", ns);
|
|
ns = ",";
|
|
|
|
if (pos > 72) {
|
|
fprintf(fp, "\n");
|
|
pos = 0;
|
|
}
|
|
|
|
pos += fprintf(fp, " 0x%02X", t->t_token);
|
|
}
|
|
|
|
fprintf(fp, "\n};\n", nstates);
|
|
}
|
|
|
|
|
|
main(argc, argv)
|
|
int argc;
|
|
char * * argv;
|
|
|
|
{
|
|
FILE * fp;
|
|
t_chset * csp;
|
|
char symbuf[20];
|
|
|
|
chset_list = (t_chset *) NULL;
|
|
initial_state = newstate();
|
|
job2utf8 = iconv_open_ccsid(C_UTF8_CCSID, C_SOURCE_CCSID, 0);
|
|
utf82job = iconv_open_ccsid(C_SOURCE_CCSID, C_UTF8_CCSID, 0);
|
|
|
|
if (argc != 4) {
|
|
fprintf(stderr, "Usage: %s <ccsid-mibenum file> ", *argv);
|
|
fprintf(stderr, "<iana-character-set file> <output file>\n");
|
|
exit(1);
|
|
}
|
|
|
|
/**
|
|
*** Read CCSID/MIBenum associations. Define special names.
|
|
**/
|
|
|
|
read_assocs(argv[1]);
|
|
|
|
/**
|
|
*** Read character set names and establish the case-independent
|
|
*** name DFA in all possible CCSIDs.
|
|
**/
|
|
|
|
read_iana(argv[2]);
|
|
|
|
/**
|
|
*** Build DFA from NFA.
|
|
**/
|
|
|
|
builddfa();
|
|
|
|
/**
|
|
*** Delete NFA.
|
|
**/
|
|
|
|
deletenfa();
|
|
|
|
/**
|
|
*** Minimize the DFA state count.
|
|
**/
|
|
|
|
optimizedfa();
|
|
|
|
/**
|
|
*** Generate the table.
|
|
**/
|
|
|
|
fp = fopen(argv[3], "w+");
|
|
|
|
if (!fp) {
|
|
perror(argv[3]);
|
|
exit(1);
|
|
}
|
|
|
|
fprintf(fp, "/**\n");
|
|
fprintf(fp, "*** Character set names table.\n");
|
|
fprintf(fp, "*** Generated by program BLDCSNDFA from");
|
|
fprintf(fp, " IANA character set assignment file\n");
|
|
fprintf(fp, "*** and CCSID/MIBenum equivalence file.\n");
|
|
fprintf(fp, "*** *** Do not edit by hand ***\n");
|
|
fprintf(fp, "**/\n\n");
|
|
listids(fp);
|
|
generate(fp);
|
|
|
|
if (ferror(fp)) {
|
|
perror(argv[3]);
|
|
fclose(fp);
|
|
exit(1);
|
|
}
|
|
|
|
fclose(fp);
|
|
iconv_close(job2utf8);
|
|
iconv_close(utf82job);
|
|
exit(0);
|
|
}
|