// Copyright 1997-1999 Omni Development, Inc.  All rights reserved.
//
// This software may only be used and reproduced according to the
// terms in the file OmniSourceLicense.html, which should be
// distributed with this project and can also be found at
// http://www.omnigroup.com/DeveloperResources/OmniSourceLicense.html.

#import "OWHTMLToSGMLObjects.h"

#import <Foundation/Foundation.h>
#import <OmniBase/OmniBase.h>
#import <OmniFoundation/OmniFoundation.h>

#import "NSString-OWSGMLString.h"
#import "OWSGMLTag.h"
#import "OWSGMLTagType.h"
#import "OWSGMLAttribute.h"
#import "OWSGMLDTD.h"
#import "OWObjectStream.h"
#import "OWDataStream.h"
#import "OWDataStreamCursor.h"
#import "OWDataStreamScanner.h"
#import "OWPipeline.h"

RCS_ID("$Header: /Network/Source/CVS/OmniGroup/Frameworks/OWF/Processors.subproj/SGML.subproj/OWHTMLToSGMLObjects.m,v 1.25 1999/07/27 23:43:54 bungi Exp $")

@interface OWDataStreamScanner (SpecialScanning)
- (NSString *)readFragmentUpToLeftAngleBracketOrAmpersand;
- (NSString *)readFragmentUpThroughCharacters:(unichar *)terminator alternative:(unichar *)alterminator length:(int)terminatorLength seen:(int *)terminatorSeen;
@end

@interface OWHTMLToSGMLObjects (HTMLParse)
- (void)scanContent;
- (void)scanTag;
- (void)scanBeginTag;
- (NSString *)readValueWithDelimiterCSBitmap:(CSBitmap)delimiterCSBitmap newlinesAreDelimiters:(BOOL)newlinesAreDelimiters;
- (void)skipValueWithDelimiterCSBitmap:(CSBitmap)delimiterCSBitmap newlinesAreDelimiters:(BOOL)newlinesAreDelimiters;
- (void)scanEndTag;
- (void)scanMarkupDeclaration;
- (void)scanComment;
- (id <OWSGMLToken>)readEntity;
- (id <OWSGMLToken>)readCharacterReference;
- (id <OWSGMLToken>)readEntityReference;
- (unsigned int)readNumber;
- (unsigned int)readHexNumber;
- (void)skipToEndOfTag;
- (void)scanNonSGMLContent:(OWSGMLTag *)nonSGMLTag;
@end

@interface OWHTMLToSGMLObjects (Private)
+ (void)readDefaults;
+ (void)readEncodingDefaults;
@end

@implementation OWHTMLToSGMLObjects

static BOOL mosaicCompatibleNonterminatedQuotedValues = NO;
static BOOL netscapeCompatibleComments = YES;
static BOOL netscapeCompatibleNewlineAfterEntity = YES;
    // ISO 8879 9.4.5 [61] 353:1 says that newlines following an entity should be ignored.  Unfortunately, Netscape preserves them.
static BOOL netscapeCompatibleNonterminatedEntities = NO;
    // Netscape does not require proper termination of entities.

static Class stringDecoderClass;
static NSString *decoderMode;
static NSStringEncoding stringEncoding;
static unsigned int encodingChangeCount = NSNotFound;
static unsigned int decodingChangeCount = NSNotFound;
NSLock *decoderDefaultsLock = nil;
 
static NSDictionary *entityDictionary;
static NSMutableDictionary *stringEntityDictionary;

// abstract syntax

static NSCharacterSet *DigitSet;
static NSCharacterSet *InvertedDigitSet;
static NSMutableCharacterSet *InvertedHexDigitSet;
static NSCharacterSet *LCLetterSet;
static NSCharacterSet *UCLetterSet;
static NSCharacterSet *SpecialSet;

// concrete syntax

static NSCharacterSet *LCNameCharSet;
static NSCharacterSet *RecordEndSet;
static NSCharacterSet *RecordStartSet;
static NSCharacterSet *SepCharSet;
static NSCharacterSet *SpaceSet;
static NSCharacterSet *UCNameCharSet;

// categories

static NSMutableCharacterSet *NameStartCharacterSet;
static NSMutableCharacterSet *InvertedNameCharacterSet;
static NSMutableCharacterSet *BlankSpaceSet;
static NSCharacterSet *InvertedBlankSpaceSet;
static NSMutableCharacterSet *CREFSet;

// made up

static NSCharacterSet *CommentEndSet;
static NSCharacterSet *ContentEndSet;
static NSCharacterSet *EndQuotedValueSet;
static NSCharacterSet *EndSingleQuotedValueSet;
static NSCharacterSet *EndQuotedValueSetStandard;
static NSCharacterSet *EndSingleQuotedValueSetStandard;
static NSCharacterSet *EndQuotedValueSetAlaMosaic;
static NSCharacterSet *EndSingleQuotedValueSetAlaMosaic;
static NSCharacterSet *EndTagSet;
static NSCharacterSet *EndTagSetStandard;
static NSCharacterSet *EndTagSetAlaMosaic;
static NSMutableCharacterSet *EndValueSet;
static NSMutableCharacterSet *TagEndOrNameStartCharacterSet;

// bitmaps

static CSBitmap CREFCSBitmap;
static CSBitmap CommentEndCSBitmap;
static CSBitmap DigitCSBitmap;
static CSBitmap EndQuotedValueCSBitmap;
static CSBitmap EndSingleQuotedValueCSBitmap;
static CSBitmap EndTagCSBitmap;
static CSBitmap EndValueCSBitmap;
static CSBitmap InvertedBlankSpaceCSBitmap;
static CSBitmap InvertedDigitCSBitmap;
static CSBitmap InvertedHexDigitCSBitmap;
static CSBitmap InvertedNameCSBitmap;
static CSBitmap NameStartCSBitmap;
static CSBitmap TagEndOrNameStartCSBitmap;


+ (void)initialize;
{
    static BOOL initialized = NO;
    NSDictionary *characterDictionary;
    NSEnumerator *characterKeyEnumerator;
    NSString *name, *value;
    NSAutoreleasePool *pool;

    [super initialize];
    if (initialized)
        return;
    initialized = YES;

    pool = [[NSAutoreleasePool alloc] init];

    entityDictionary = [[NSDictionary alloc] initWithContentsOfFile:[[NSBundle bundleForClass:self] pathForResource:@"entities" ofType:@"plist"]];
    stringEntityDictionary = [[entityDictionary objectForKey:@"strings"] mutableCopy];
    characterDictionary = [entityDictionary objectForKey:@"character"];
    characterKeyEnumerator = [characterDictionary keyEnumerator];
    while ((name = [characterKeyEnumerator nextObject])) {
	unichar character;

	value = [characterDictionary objectForKey:name];
	character = [value intValue];
	value = [NSString stringWithCharacters:&character length:1];
	[stringEntityDictionary setObject:value forKey:name];
    }
    
    if (!decoderDefaultsLock)
        decoderDefaultsLock = [[NSLock alloc] init];

// abstract syntax

    DigitSet = [[NSCharacterSet decimalDigitCharacterSet] retain];
    InvertedDigitSet = [[DigitSet invertedSet] retain];

    InvertedHexDigitSet = [[NSMutableCharacterSet alloc] init];
    [InvertedHexDigitSet formUnionWithCharacterSet:DigitSet];
    [InvertedHexDigitSet addCharactersInString:@"abcdefABCDEF"];
    [InvertedHexDigitSet invert];
    
    LCLetterSet = [[NSCharacterSet lowercaseLetterCharacterSet] retain];
    UCLetterSet = [[NSCharacterSet uppercaseLetterCharacterSet] retain];
    SpecialSet = [[NSCharacterSet characterSetWithCharactersInString:@"'()+,-./:=?"] retain];

// concrete syntax

    LCNameCharSet = [[NSCharacterSet characterSetWithCharactersInString:@"-."] retain];
    RecordEndSet = [[NSCharacterSet characterSetWithCharactersInString:@"\n"] retain];
    RecordStartSet = [[NSCharacterSet characterSetWithCharactersInString:@"\r"] retain];
    SepCharSet = [[NSCharacterSet characterSetWithCharactersInString:@"\t"] retain];
    SpaceSet = [[NSCharacterSet characterSetWithCharactersInString:@" "] retain];
    UCNameCharSet = [[NSCharacterSet characterSetWithCharactersInString:@"-."] retain];

// categories

    NameStartCharacterSet = [[NSMutableCharacterSet alloc] init];
    [NameStartCharacterSet formUnionWithCharacterSet:LCLetterSet];
    [NameStartCharacterSet formUnionWithCharacterSet:UCLetterSet];

    InvertedNameCharacterSet = [[NSMutableCharacterSet alloc] init];
    [InvertedNameCharacterSet formUnionWithCharacterSet:NameStartCharacterSet];
    [InvertedNameCharacterSet formUnionWithCharacterSet:DigitSet];
    [InvertedNameCharacterSet formUnionWithCharacterSet:LCNameCharSet];
    [InvertedNameCharacterSet formUnionWithCharacterSet:UCNameCharSet];
    [InvertedNameCharacterSet invert];

    BlankSpaceSet = [[NSMutableCharacterSet alloc] init];
    [BlankSpaceSet formUnionWithCharacterSet:SpaceSet];
    [BlankSpaceSet formUnionWithCharacterSet:RecordEndSet];
    [BlankSpaceSet formUnionWithCharacterSet:RecordStartSet];
    [BlankSpaceSet formUnionWithCharacterSet:SepCharSet];
    InvertedBlankSpaceSet = [[BlankSpaceSet invertedSet] retain];

    CREFSet = [[NSMutableCharacterSet alloc] init];
    [CREFSet formUnionWithCharacterSet:DigitSet];
    [CREFSet addCharactersInString:@"xX"]; // SGML allows others, HTML does not

// made up

    CommentEndSet = [[NSCharacterSet characterSetWithCharactersInString:@"-"] retain];
    ContentEndSet = [[NSCharacterSet characterSetWithCharactersInString:@"<&"] retain];
    EndQuotedValueSetStandard = [[NSCharacterSet characterSetWithCharactersInString:@"&\"\r\n"] retain];
    EndSingleQuotedValueSetStandard = [[NSCharacterSet characterSetWithCharactersInString:@"&'\r\n"] retain];
    EndQuotedValueSetAlaMosaic = [[NSCharacterSet characterSetWithCharactersInString:@"&\">\r\n"] retain];
    EndSingleQuotedValueSetAlaMosaic = [[NSCharacterSet characterSetWithCharactersInString:@"&'>\r\n"] retain];
    EndQuotedValueSet = EndQuotedValueSetStandard;
    EndSingleQuotedValueSet = EndSingleQuotedValueSetStandard;

    EndValueSet = [BlankSpaceSet mutableCopy];
    [EndValueSet addCharactersInString:@"&>"];

    EndTagSetStandard = [[NSCharacterSet characterSetWithCharactersInString:@">'\""] retain];
    EndTagSetAlaMosaic = [[NSCharacterSet characterSetWithCharactersInString:@">"] retain];
    EndTagSet = EndTagSetStandard;

    TagEndOrNameStartCharacterSet = [NameStartCharacterSet mutableCopy];
    [TagEndOrNameStartCharacterSet addCharactersInString:@">"];

    // Setup bitmaps

#define bitmapForSet(aSet) bitmapForCharacterSetDoRetain(aSet, YES);
    
    CommentEndCSBitmap = bitmapForSet(CommentEndSet);
    CREFCSBitmap = bitmapForSet(CREFSet);
    DigitCSBitmap = bitmapForSet(DigitSet);
    EndQuotedValueCSBitmap = bitmapForSet(EndQuotedValueSet);
    EndSingleQuotedValueCSBitmap = bitmapForSet(EndSingleQuotedValueSet);
    EndTagCSBitmap = bitmapForSet(EndTagSet);
    EndValueCSBitmap = bitmapForSet(EndValueSet);
    InvertedBlankSpaceCSBitmap = bitmapForSet(InvertedBlankSpaceSet);
    InvertedDigitCSBitmap = bitmapForSet(InvertedDigitSet);
    InvertedHexDigitCSBitmap = bitmapForSet(InvertedHexDigitSet);
    InvertedNameCSBitmap = bitmapForSet(InvertedNameCharacterSet);
    NameStartCSBitmap = bitmapForSet(NameStartCharacterSet);
    TagEndOrNameStartCSBitmap = bitmapForSet(TagEndOrNameStartCharacterSet);

    [pool release];
}

+ (void)registerItemName:(NSString *)itemName bundle:(NSBundle *)bundle description:(NSDictionary *)description;
{
    // Register entity lists here
}

+ (id <OWStringDecoder>)stringDecoder;
{
    [self readEncodingDefaults];
    if (stringDecoderClass && decoderMode)
	return [[[stringDecoderClass alloc] initWithMode:decoderMode] autorelease];
    return nil;
}

+ (NSStringEncoding)stringEncoding;
{
    [self readEncodingDefaults];

    // Note that 0 is guaranteed never to be a valid encoding by the semantics of +[NSString availableStringEncodings], so we use it to indicate a default encoding setting.
    return stringEncoding == 0 ? NSWindowsCP1252StringEncoding : stringEncoding;
}

+ (void)setStringDecoderClass:(Class)aClass;
{
    stringDecoderClass = aClass;
}

+ (Class)stringDecoderClass;
{
    return stringDecoderClass;
}

+ (BOOL)recognizesEntityNamed:(NSString *)entityName;
{
    return [stringEntityDictionary objectForKey:entityName] != nil;
}

// Init and dealloc

- initWithDataStreamCursor:(OWDataStreamCursor *)aDataStreamCursor forDTD:(OWSGMLDTD *)aDTD;
{
    NSObject <OWStringDecoder> *decoder;

    [isa readDefaults];

    if (dataCursor != aDataStreamCursor) {
        if (dataCursor)
            [dataCursor release];
        dataCursor = [aDataStreamCursor retain];
    }
    objectStream = [[OWObjectStream alloc] init];
    tagTrie = [aDTD tagTrie];
    [objectStream setContentType:[aDTD destinationType]];
    decoder = [[self class] stringDecoder];
    if (decoder)
        [dataCursor setStringDecoder:decoder];
    else if ([[self class] stringEncoding])
        [dataCursor setStringEncoding:[[self class] stringEncoding]];
    scanner = [[OWDataStreamScanner alloc] initWithCursor:dataCursor];
    [pipeline addContent:objectStream];
    [pipeline cacheContent];
    [pipeline startProcessingContent];
    return self;
}

- initWithDataStreamCursor:(OWDataStreamCursor *)aDataStreamCursor;
{
    OWSGMLDTD *sourceContentDTD;

    sourceContentDTD = [OWSGMLDTD dtdForSourceContentType:[[aDataStreamCursor dataStream] encodedContentType]];
    return [self initWithDataStreamCursor:aDataStreamCursor forDTD:sourceContentDTD];
}

- initWithPipeline:(OWPipeline *)aPipeline;
{
    if (![super initWithPipeline:aPipeline])
        return nil;
    return [self initWithDataStreamCursor:dataCursor];
}

- (void)dealloc;
{
    [objectStream release];
    [scanner release];
    [super dealloc];
}

//

- (OWObjectStream *)outputStream;
{
    return objectStream;
}

// OWProcessor subclass

- (void)process;
{
#ifdef KANJI
    [[dataCursor dataStream] waitForDataEnd];
#endif
    [self scanContent];
}

- (void)processAbort;
{
    [objectStream dataAbort];
    [super processAbort];
}

// Debugging

- (NSMutableDictionary *)debugDictionary;
{
    NSMutableDictionary        *debugDictionary;

    debugDictionary = [super debugDictionary];
    if (objectStream)
	[debugDictionary setObject:objectStream forKey:@"objectStream"];
    if (scanner)
	[debugDictionary setObject:scanner forKey:@"scanner"];
    return debugDictionary;
}

@end

@implementation OWDataStreamScanner (SpecialScanning)

- (NSString *)readFragmentUpToLeftAngleBracketOrAmpersand;
{
    unichar *startLocation;

    if (!scannerHasData(self))
	return nil;

    startLocation = scanLocation;
    while (scanLocation < scanEnd) {
	if (*scanLocation == '<' || *scanLocation == '&')
	    break;
	scanLocation++;
    }
    return [NSString stringWithCharacters:startLocation length:scanLocation - startLocation];
}

// TODO: Ideally, want a version of this that returns NSDatas. Semantics problematic though.  Also, this method interface sucks.
- (NSString *)readFragmentUpThroughCharacters:(unichar *)terminator alternative:(unichar *)alterminator length:(int)terminatorLength seen:(int *)terminatorSeen
{
    unichar *startLocation;
    int terminatorProgress;

    if (!scannerHasData(self))
        return nil;
    
    startLocation = scanLocation;

    terminatorProgress = *terminatorSeen;
    while (scanLocation < scanEnd && terminatorProgress < terminatorLength) {
        if (*scanLocation == terminator[terminatorProgress] ||
            *scanLocation == alterminator[terminatorProgress])
            terminatorProgress ++;
        else
            terminatorProgress = 0;
        scanLocation++;
    }
    *terminatorSeen = terminatorProgress;
    
    return [NSString stringWithCharacters:startLocation
            length:scanLocation - startLocation];
}

@end

@implementation OWHTMLToSGMLObjects (HTMLParse)

#ifdef DEBUG
static BOOL OWHTMLToSGMLObjectsDebug = NO;

+ (void)setDebug:(BOOL)newDebug;
{
    OWHTMLToSGMLObjectsDebug = newDebug;
}
#endif

- (void)scanContent;
{
    unichar peekCharacter;
    NSString *string;

    if (!scanner)
	return;
    while ((peekCharacter = scannerPeekCharacter(scanner)) != OFStringScannerEndOfDataCharacter || scannerHasData(scanner) /* In case OFStringScannerEndOfDataCharacter is an actual character in the string */) {
        switch (peekCharacter) {
            case '<':
                scannerSkipPeekedCharacter(scanner);
                [self scanTag];
                break;
            case '&':
                scannerSkipPeekedCharacter(scanner);
                [objectStream writeObject:[self readEntity]];
                break;
            default:
                string = [scanner readFragmentUpToLeftAngleBracketOrAmpersand];
                [objectStream writeObject:string];
                break;
        }
    }

    if ([dataCursor stringDecoder]) {
	NSStringEncoding dominantEncoding;

	dominantEncoding = [[dataCursor stringDecoder] dominantEncoding];
	if (dominantEncoding != 0 && dominantEncoding != ~(unsigned)0) {
	    [pipeline setContextObject:[NSNumber numberWithInt:dominantEncoding] forKey:@"dominantCharacterEncoding"];
	}
    }

    [objectStream dataEnd];
}

- (void)scanTag;
{
    unichar peekCharacter;

    switch ((peekCharacter = scannerPeekCharacter(scanner))) {
        case '/':
            scannerSkipPeekedCharacter(scanner);
            [self scanEndTag];
            break;
        case '!':
            scannerSkipPeekedCharacter(scanner);
            [self scanMarkupDeclaration];
            break;
        default:
            if (characterIsMemberOfCSBitmap(NameStartCSBitmap, peekCharacter))
                [self scanBeginTag];
            else
                [objectStream writeObject:@"<"];
            break;
    }
}

- (void)scanBeginTag;
{
    OWSGMLTagType *tagType;
    OWSGMLTag *tag = nil;
    OFTrie *attributeTrie;
    OWSGMLAttribute *attribute;

    tagType = (OWSGMLTagType *)[scanner readLongestTrieElement:tagTrie];
    if (!tagType || !characterIsMemberOfCSBitmap(InvertedNameCSBitmap, scannerPeekCharacter(scanner))) {
	[self skipToEndOfTag];
	return;
    }

    attributeTrie = [tagType attributeTrie];
    
    while (scannerHasData(scanner)) {
	NSString *value;

	scannerScanUpToCharacterInCSBitmap(scanner, TagEndOrNameStartCSBitmap);
	if (scannerPeekCharacter(scanner) == '>') {
            scannerReadCharacter(scanner);
	    break;
	}
                           
	attribute = (OWSGMLAttribute *)[scanner readLongestTrieElement:attributeTrie];

	if (attribute && characterIsMemberOfCSBitmap(InvertedNameCSBitmap, scannerPeekCharacter(scanner))) {
	    scannerScanUpToCharacterInCSBitmap(scanner, InvertedBlankSpaceCSBitmap);
	    if (scannerPeekCharacter(scanner) == '=') {
		unichar character;

		scannerSkipPeekedCharacter(scanner);
		scannerScanUpToCharacterInCSBitmap(scanner, InvertedBlankSpaceCSBitmap);

                switch ((character = scannerPeekCharacter(scanner))) {
                    case '"':
                    case '\'':
                        scannerSkipPeekedCharacter(scanner);
                        value = [self readValueWithDelimiterCSBitmap:(character == '"' ? EndQuotedValueCSBitmap : EndSingleQuotedValueCSBitmap) newlinesAreDelimiters:NO];
                        if (scannerPeekCharacter(scanner) != '>')
                            scannerSkipPeekedCharacter(scanner);
                            break;
                    default:
                        value = [self readValueWithDelimiterCSBitmap:EndValueCSBitmap newlinesAreDelimiters:YES];
                        break;
                }
	    } else {
		value = [OFNull nullStringObject];
	    }
            if (!tag)
                tag = [[OWSGMLTag alloc] initWithTokenType:OWSGMLTokenTypeStartTag tagType:tagType];
	    [tag setValue:value atIndex:[attribute offset]];
	} else {
            scannerScanUpToCharacterInCSBitmap(scanner, InvertedNameCSBitmap);
            scannerScanUpToCharacterInCSBitmap(scanner, InvertedBlankSpaceCSBitmap);
            if (scannerPeekCharacter(scanner) == '=') {
                unichar character;

                scannerSkipPeekedCharacter(scanner);
                scannerScanUpToCharacterInCSBitmap(scanner, InvertedBlankSpaceCSBitmap);

                switch ((character = scannerPeekCharacter(scanner))) {
                    case '"':
                    case '\'':
                        scannerSkipPeekedCharacter(scanner);
                        [self skipValueWithDelimiterCSBitmap:(character == '"' ? EndQuotedValueCSBitmap : EndSingleQuotedValueCSBitmap) newlinesAreDelimiters:NO];
                        if (scannerPeekCharacter(scanner) != '>')
                            scannerSkipPeekedCharacter(scanner);
                            break;
                    default:
                        [self skipValueWithDelimiterCSBitmap:EndValueCSBitmap newlinesAreDelimiters:NO];
                        break;
                }
            }
        }
    }

    if (!tag)
        tag = [tagType attributelessStartTag];
    [objectStream writeObject:tag];
#ifdef DEBUG
    if (OWHTMLToSGMLObjectsDebug)
	NSLog(@"Tag: %@", tag);
#endif

    // Ugly hack to support non-SGML tags such as <SCRIPT> and stylesheets
    if ([tagType isNotValidSGML])
        [self scanNonSGMLContent:tag];

    if (tag != [tagType attributelessStartTag])
        [tag release];
}

- (NSString *)readValueWithDelimiterCSBitmap:(CSBitmap)delimiterCSBitmap
  newlinesAreDelimiters:(BOOL)newlinesAreDelimiters;
{
    NSMutableString *mergedValue;
    NSString *value;
    BOOL stillLooking = YES;

    mergedValue = nil;
    value = nil;
    while (stillLooking) {
	id <OWSGMLToken> entityToken;
	unichar peekCharacter;
	    
	value = [scanner readFullTokenWithDelimiterCSBitmap:delimiterCSBitmap forceLowercase:NO];
	if (mergedValue)
	    [mergedValue appendString:value];

	peekCharacter = scannerPeekCharacter(scanner);
        switch (peekCharacter) {
            case '&':
                scannerSkipPeekedCharacter(scanner);
                entityToken = [self readEntity];
                if (!mergedValue)
                    mergedValue = [[value mutableCopy] autorelease];
                    [mergedValue appendString:[entityToken string]];
                break;
            case '\r':
            case '\n':
                if (newlinesAreDelimiters) {
                    stillLooking = NO;
                    break;
                }

                // True SGML would have us replace these with whitespace, but, for now, we just ignore them (ala Netscape 2.0).

                do {
                    scannerSkipPeekedCharacter(scanner);
                    peekCharacter = scannerPeekCharacter(scanner);
                } while (peekCharacter == '\r' || peekCharacter == '\n');
                break;
            default:
                stillLooking = NO;
                break;
        }
	if (stillLooking && !mergedValue)
	    mergedValue = [[value mutableCopy] autorelease];
    }
    
    if (mergedValue)
	return mergedValue;
    else
	return value;
}

- (void)skipValueWithDelimiterCSBitmap:(CSBitmap)delimiterCSBitmap newlinesAreDelimiters:(BOOL)newlinesAreDelimiters;
{
    BOOL stillLooking = YES;

    while (stillLooking) {
        id <OWSGMLToken> entityToken;
        unichar peekCharacter;

        scannerScanUpToCharacterInCSBitmap(scanner, delimiterCSBitmap);
        peekCharacter = scannerPeekCharacter(scanner);
        switch (peekCharacter) {
            case '&':
                scannerSkipPeekedCharacter(scanner);
                entityToken = [self readEntity];
                break;

            case '\r':
            case '\n':
                if (newlinesAreDelimiters) {
                    stillLooking = NO;
                    break;
                }

                do {
                    scannerSkipPeekedCharacter(scanner);
                    peekCharacter = scannerPeekCharacter(scanner);
                } while (peekCharacter == '\r' || peekCharacter == '\n');
                break;

            default:
                stillLooking = NO;
                break;
        }
    }
}

- (void)scanEndTag;
{
    OWSGMLTagType *tagType;

    if (!characterIsMemberOfCSBitmap(NameStartCSBitmap, scannerPeekCharacter(scanner))) {
        [objectStream writeObject:@"</"];
        return;
    }

    tagType = (OWSGMLTagType *)[scanner readLongestTrieElement:tagTrie];
    if (tagType && characterIsMemberOfCSBitmap(InvertedNameCSBitmap, scannerPeekCharacter(scanner))) {
        [objectStream writeObject:[tagType attributelessEndTag]];
#ifdef DEBUG
        if (OWHTMLToSGMLObjectsDebug)
            NSLog(@"Tag: %@", [tagType attributelessEndTag]);
#endif
    }
    [self skipToEndOfTag];
}

- (void)scanMarkupDeclaration;
{
    unichar character;

    character = scannerReadCharacter(scanner);
    if (character == '>') {
	// Empty declaration! We're done.
    } else if (character == '-' && scannerPeekCharacter(scanner) == '-') {
	scannerSkipPeekedCharacter(scanner);
	[self scanComment];
    } else if (netscapeCompatibleComments ||
               characterIsMemberOfCSBitmap(NameStartCSBitmap, character)) {
	[self skipToEndOfTag];
    } else {
	// Not markup after all!
	[scanner skipCharacters:-1];
	[objectStream writeObject:@"<!"];
    }
}

- (void)scanComment;
{
    unsigned int commentBegin;
    unichar character;

    commentBegin = [scanner scanLocation];

    do {
        scannerScanUpToCharacterInCSBitmap(scanner, CommentEndCSBitmap);
	if (scannerReadCharacter(scanner) == '-' && scannerReadCharacter(scanner) == '-') {
	    while (scannerPeekCharacter(scanner) == '-')
		scannerSkipPeekedCharacter(scanner);
            scannerScanUpToCharacterInCSBitmap(scanner, InvertedBlankSpaceCSBitmap);
	    character = scannerPeekCharacter(scanner);
	    if (character == '>') {
		scannerSkipPeekedCharacter(scanner);
		return;
	    }
	}
    } while (scannerHasData(scanner));

    // Woops, not a proper SGML comment!  Let's try old-style HTML.
    [scanner setScanLocation:commentBegin];
    [self skipToEndOfTag];
}

- (id <OWSGMLToken>)readEntity;
{
    unichar character;
    
    character = scannerPeekCharacter(scanner);
    if (character == '#') {
	scannerSkipPeekedCharacter(scanner);
	return [self readCharacterReference];
    } else if (characterIsMemberOfCSBitmap(NameStartCSBitmap, character))
	return [self readEntityReference];
    else
	return @"&";
}

- (id <OWSGMLToken>)readCharacterReference;
{
    NSString *value;
    unichar character;

    character = scannerPeekCharacter(scanner);
    if (!characterIsMemberOfCSBitmap(CREFCSBitmap, character))
	return @"&#";

    if (characterIsMemberOfCSBitmap(DigitCSBitmap, character)) {
	character = [self readNumber];
    } else { // character is 'x'
        scannerSkipPeekedCharacter(scanner);
        character = [self readHexNumber];
   }

    // WJS: 5/19/98 Even though the upper control characters aren't mapped in ISO Latin-1, they work in Netscape and Windows, so we check for that range explicitly and interpret them as WindowsCP1252 characters.  Note that right now the NSString machinery turns 145 and 146 into single-quotes that don't draw very well in NSText (they aren't mapped to the right glyphs in their font), so we special-case those two and turn them into quotes that draw nicely.
    if (character == 145)
        character = 96; // unicode GRAVE ACCENT
    else if (character == 146)
        character = 39; // unicode APOSTROPHE

    if (character > 0x7e && character < 0xa0) {
        unsigned char byte;
        NSData *data;

        byte = character & 0xff;
        data = [[NSData alloc] initWithBytes:&byte length:1];
        value = [[[NSString alloc] initWithData:data encoding:NSWindowsCP1252StringEncoding] autorelease];
        [data release];
    } else {
        value = [NSString stringWithCharacters:&character length:1];
    }

    character = scannerPeekCharacter(scanner);
    if (character == ';' || (!netscapeCompatibleNewlineAfterEntity && character == '\n'))
	scannerSkipPeekedCharacter(scanner);
    return value;
}

- (id <OWSGMLToken>)readEntityReference;
{
    NSString *name, *value;
    unsigned int nameLength;

    name = [scanner readFullTokenWithDelimiterCSBitmap:InvertedNameCSBitmap forceLowercase:NO];
    nameLength = [name length];
    if (nameLength == 0)
        return @"&";

    value = [stringEntityDictionary objectForKey:name];
    if (value) {
        unichar character;

        character = scannerPeekCharacter(scanner);
        if (character == ';' || (character == '\n' && !netscapeCompatibleNewlineAfterEntity))
            scannerSkipPeekedCharacter(scanner);
        return value;
    } else {
	if (netscapeCompatibleNonterminatedEntities) {
	    unsigned int tryLength;

	    for (tryLength = nameLength - 1; tryLength > 0; tryLength--) {
		value = [stringEntityDictionary objectForKey:[name substringToIndex:tryLength]];
		if (value) {
		    [scanner skipCharacters:-(int)(nameLength - tryLength)];
		    return value;
		}
	    }
	}
	return [NSString stringWithFormat:@"&%@", name];
    }
}

- (unsigned int)readNumber;
{
    return [[scanner readFullTokenWithDelimiterCSBitmap:InvertedDigitCSBitmap forceLowercase:NO] intValue];
}

- (unsigned int)readHexNumber;
{
    return [[scanner readFullTokenWithDelimiterCSBitmap:InvertedHexDigitCSBitmap forceLowercase:NO] hexValue];
}

- (void)skipToEndOfTag;
{
    unsigned int originalLocation;

    originalLocation = scannerScanLocation(scanner);
    for (;;) {
        unichar character;

        if (!scannerScanUpToCharacterInCSBitmap(scanner, EndTagCSBitmap))
            break; // abort

        character = scannerReadCharacter(scanner);
        switch (character) {
            case '>':
                return; // success
            default:
                // find matching quote
                if (scannerScanUpToCharacter(scanner, character))
                    scannerSkipPeekedCharacter(scanner);
                break;
        }
    }

    // Fine, I give up!
    [scanner setScanLocation:originalLocation];
}

- (void)scanNonSGMLContent:(OWSGMLTag *)nonSGMLTag;
{
    unichar *uppercaseEndTag, *lowercaseEndTag;
    unsigned int endTagLength;
    unsigned int endTagConsumed;
    NSString *endTagString;
    OWSGMLTag *endTag;
    NSString *fragment, *deferredFragment;

#warning -scanNonSGMLContent: should really scan NSData
    // Non-SGML content is probably best interpreted as raw bytes instead of whatever character encoding the document happens to be in.  However, AFAIK JavaScript doesn't handle non-ASCII characters so this isn't immediately urgent.

    if (!scanner)
        return;

    endTagString = [NSString stringWithFormat:@"</%@", [[nonSGMLTag name] lowercaseString]];
    endTagLength = [endTagString length];
    OBASSERT(endTagLength < 1024); // tag names longer than 1k?!?
    lowercaseEndTag = alloca(endTagLength * sizeof(unichar));
    [endTagString getCharacters:lowercaseEndTag];
    endTagString = [endTagString uppercaseString];
    OBASSERT(endTagLength == [endTagString length]);
    uppercaseEndTag = alloca(endTagLength * sizeof(unichar));
    [endTagString getCharacters:uppercaseEndTag];

    endTagConsumed = 0;
    deferredFragment = nil;
    while (endTagConsumed < endTagLength && scannerHasData(scanner)) {
        fragment = [scanner readFragmentUpThroughCharacters:lowercaseEndTag alternative:uppercaseEndTag length:endTagLength seen:&endTagConsumed];

        // If endTagConsumed is > 0, then the last few characters of "fragment" might be the beginning of the end tag, so we shouldn't pass them along to the output stream. But they might not be, so we have to keep them around in deferredFragment and pass them along if, on the next iteration, it turns out to have been a false alarm.
        if (deferredFragment) {
            if ([fragment length] > endTagConsumed) {
                [objectStream writeObject:deferredFragment];
                [deferredFragment release];
                deferredFragment = nil;
            } else {
                fragment = [deferredFragment stringByAppendingString:fragment];
                [deferredFragment release];
                deferredFragment = nil;
            }
        }

        if (endTagConsumed > 0) {
            unsigned int possiblyTheBeginningOfTheEnd;

            possiblyTheBeginningOfTheEnd = [fragment length] - endTagConsumed;
            deferredFragment = [fragment substringFromIndex:possiblyTheBeginningOfTheEnd];
            [deferredFragment retain];
            fragment = [fragment substringToIndex:possiblyTheBeginningOfTheEnd];
        }

        [objectStream writeObject:fragment];
#ifdef DEBUG
        if (OWHTMLToSGMLObjectsDebug)
            NSLog(@"NonSGMLFragment: %@", fragment);
        NSLog(@"NonSGML: length=%d consumed=%d", endTagLength, endTagConsumed);
#endif
    }

    [deferredFragment release];

    
    [self skipToEndOfTag];

    // Create an end tag and write it to the stream (since we just skipped over the real one).
    endTag = [[OWSGMLTag alloc] initWithTokenType:OWSGMLTokenTypeEndTag tagType:[nonSGMLTag tagType]];
    [objectStream writeObject:endTag];
#ifdef DEBUG
    if (OWHTMLToSGMLObjectsDebug)
        NSLog(@"Tag: %@", endTag);
#endif
    [endTag release];
}
    
@end

@implementation OWHTMLToSGMLObjects (Private)

+ (void)readDefaults;
{
    OFUserDefaults *userDefaults;

    userDefaults = [OFUserDefaults sharedUserDefaults];

    [decoderDefaultsLock lock];

    if ([userDefaults changeCount] == decodingChangeCount) {
        [decoderDefaultsLock unlock];
        return;
    }

    mosaicCompatibleNonterminatedQuotedValues = [userDefaults boolForKey:@"OWHTMLMosaicCompatibleNonterminatedQuotedValues"];
    netscapeCompatibleComments = [userDefaults boolForKey:@"OWHTMLNetscapeCompatibleComments"];
    netscapeCompatibleNewlineAfterEntity = [userDefaults boolForKey:@"OWHTMLNetscapeCompatibleNewlineAfterEntity"];
    netscapeCompatibleNonterminatedEntities = [userDefaults boolForKey:@"OWHTMLNetscapeCompatibleNonterminatedEntities"];

    if (mosaicCompatibleNonterminatedQuotedValues) {
        EndQuotedValueSet = EndQuotedValueSetAlaMosaic;
        EndSingleQuotedValueSet = EndSingleQuotedValueSetAlaMosaic;
        EndTagSet = EndTagSetAlaMosaic;
    } else {
        EndQuotedValueSet = EndQuotedValueSetStandard;
        EndSingleQuotedValueSet = EndSingleQuotedValueSetStandard;
        EndTagSet = EndTagSetStandard;
    }

#warning This is a small memory leak
    // There's no way to free the memory returned by bitmapForSet() since it's actually the -bytes of a NSData we don't have a pointer to anymore. Fortunately this code is only executed when the defaults change so this isn't a large leak.
    EndQuotedValueCSBitmap = bitmapForSet(EndQuotedValueSet);
    EndSingleQuotedValueCSBitmap = bitmapForSet(EndSingleQuotedValueSet);
    EndTagCSBitmap = bitmapForSet(EndTagSet);

    decodingChangeCount = [userDefaults changeCount];
    [decoderDefaultsLock unlock];
}

+ (void)readEncodingDefaults;
{
    OFUserDefaults *defaults;
    unsigned int newEncoding;

    defaults = [OFUserDefaults sharedUserDefaults];

    [decoderDefaultsLock lock];
    if ([defaults changeCount] == encodingChangeCount) {
        [decoderDefaultsLock unlock];
        return;
    }

    NS_DURING {
        NSScanner *modeScanner;

        [decoderMode release];
        decoderMode = nil;
        newEncoding = 0;

        modeScanner = [NSScanner scannerWithString:[defaults stringForKey:@"OWIncomingStringEncoding"]];
        if (modeScanner && ![modeScanner scanInt:&newEncoding]) {
            if (stringDecoderClass && [modeScanner scanString:[stringDecoderClass description] intoString:NULL]) {
                [modeScanner scanString:@" " intoString:NULL];
                [modeScanner scanUpToCharactersFromSet:[NSCharacterSet illegalCharacterSet] intoString:&decoderMode];
                if (![[stringDecoderClass filterModes] objectForKey:decoderMode]) {
                    decoderMode = nil;
                } else {
                    [decoderMode retain];
                }
            }
        }
        stringEncoding = newEncoding;
    } NS_HANDLER {
        decoderMode = nil;
        stringEncoding = 0;
        [decoderDefaultsLock unlock];
        [localException raise];
    } NS_ENDHANDLER;

    encodingChangeCount = [defaults changeCount];
    [decoderDefaultsLock unlock];
}

@end
