website hosted
This commit is contained in:
root
2
node_modules/sitemap/dist/cli.d.ts
generated
vendored
Normal file
2
node_modules/sitemap/dist/cli.d.ts
generated
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
#!/usr/bin/env node
|
||||
export {};
|
||||
145
node_modules/sitemap/dist/cli.js
generated
vendored
Executable file
145
node_modules/sitemap/dist/cli.js
generated
vendored
Executable file
@@ -0,0 +1,145 @@
|
||||
#!/usr/bin/env node
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
const fs_1 = require("fs");
|
||||
const xmllint_1 = require("./lib/xmllint");
|
||||
const errors_1 = require("./lib/errors");
|
||||
const sitemap_parser_1 = require("./lib/sitemap-parser");
|
||||
const utils_1 = require("./lib/utils");
|
||||
const sitemap_stream_1 = require("./lib/sitemap-stream");
|
||||
const sitemap_index_stream_1 = require("./lib/sitemap-index-stream");
|
||||
const url_1 = require("url");
|
||||
const zlib_1 = require("zlib");
|
||||
const types_1 = require("./lib/types");
|
||||
/* eslint-disable-next-line @typescript-eslint/no-var-requires */
|
||||
const arg = require('arg');
|
||||
const pickStreamOrArg = (argv) => {
|
||||
if (!argv._.length) {
|
||||
return process.stdin;
|
||||
}
|
||||
else {
|
||||
return (0, fs_1.createReadStream)(argv._[0], { encoding: 'utf8' });
|
||||
}
|
||||
};
|
||||
const argSpec = {
|
||||
'--help': Boolean,
|
||||
'--version': Boolean,
|
||||
'--validate': Boolean,
|
||||
'--index': Boolean,
|
||||
'--index-base-url': String,
|
||||
'--limit': Number,
|
||||
'--parse': Boolean,
|
||||
'--single-line-json': Boolean,
|
||||
'--prepend': String,
|
||||
'--gzip': Boolean,
|
||||
'-h': '--help',
|
||||
};
|
||||
const argv = arg(argSpec);
|
||||
function getStream() {
|
||||
if (argv._ && argv._.length) {
|
||||
return (0, fs_1.createReadStream)(argv._[0]);
|
||||
}
|
||||
else {
|
||||
console.warn('Reading from stdin. If you are not piping anything in, this command is not doing anything');
|
||||
return process.stdin;
|
||||
}
|
||||
}
|
||||
if (argv['--version']) {
|
||||
/* eslint-disable-next-line @typescript-eslint/no-var-requires */
|
||||
const packagejson = require('../package.json');
|
||||
console.log(packagejson.version);
|
||||
}
|
||||
else if (argv['--help']) {
|
||||
console.log(`
|
||||
Turn a list of urls into a sitemap xml.
|
||||
Options:
|
||||
--help Print this text
|
||||
--version Print the version
|
||||
--validate Ensure the passed in file is conforms to the sitemap spec
|
||||
--index Create an index and stream that out. Writes out sitemaps along the way.
|
||||
--index-base-url Base url the sitemaps will be hosted eg. https://example.com/sitemaps/
|
||||
--limit=45000 Set a custom limit to the items per sitemap
|
||||
--parse Parse fed xml and spit out config
|
||||
--prepend=sitemap.xml Prepend the streamed in sitemap configs to sitemap.xml
|
||||
--gzip Compress output
|
||||
--single-line-json When used with parse, it spits out each entry as json rather than the whole json.
|
||||
|
||||
# examples
|
||||
|
||||
Generate a sitemap index file as well as sitemaps
|
||||
npx sitemap --gzip --index --index-base-url https://example.com/path/to/sitemaps/ < listofurls.txt > sitemap-index.xml.gz
|
||||
|
||||
Add to a sitemap
|
||||
npx sitemap --prepend sitemap.xml < listofurls.json
|
||||
|
||||
Turn an existing sitemap into configuration understood by the sitemap library
|
||||
npx sitemap --parse sitemap.xml
|
||||
|
||||
Use XMLLib to validate your sitemap (requires xmllib)
|
||||
npx sitemap --validate sitemap.xml
|
||||
`);
|
||||
}
|
||||
else if (argv['--parse']) {
|
||||
let oStream = getStream()
|
||||
.pipe(new sitemap_parser_1.XMLToSitemapItemStream({ level: types_1.ErrorLevel.THROW }))
|
||||
.pipe(new sitemap_parser_1.ObjectStreamToJSON({ lineSeparated: !argv['--single-line-json'] }));
|
||||
if (argv['--gzip']) {
|
||||
oStream = oStream.pipe((0, zlib_1.createGzip)());
|
||||
}
|
||||
oStream.pipe(process.stdout);
|
||||
}
|
||||
else if (argv['--validate']) {
|
||||
(0, xmllint_1.xmlLint)(getStream())
|
||||
.then(() => console.log('valid'))
|
||||
.catch(([error, stderr]) => {
|
||||
if (error instanceof errors_1.XMLLintUnavailable) {
|
||||
console.error(error.message);
|
||||
return;
|
||||
}
|
||||
else {
|
||||
console.log(stderr);
|
||||
}
|
||||
});
|
||||
}
|
||||
else if (argv['--index']) {
|
||||
const limit = argv['--limit'];
|
||||
const baseURL = argv['--index-base-url'];
|
||||
if (!baseURL) {
|
||||
throw new Error("You must specify where the sitemaps will be hosted. use --index-base-url 'https://example.com/path'");
|
||||
}
|
||||
const sms = new sitemap_index_stream_1.SitemapAndIndexStream({
|
||||
limit,
|
||||
getSitemapStream: (i) => {
|
||||
const sm = new sitemap_stream_1.SitemapStream();
|
||||
const path = `./sitemap-${i}.xml`;
|
||||
let ws;
|
||||
if (argv['--gzip']) {
|
||||
ws = sm.pipe((0, zlib_1.createGzip)()).pipe((0, fs_1.createWriteStream)(path));
|
||||
}
|
||||
else {
|
||||
ws = sm.pipe((0, fs_1.createWriteStream)(path));
|
||||
}
|
||||
return [new url_1.URL(path, baseURL).toString(), sm, ws];
|
||||
},
|
||||
});
|
||||
let oStream = (0, utils_1.lineSeparatedURLsToSitemapOptions)(pickStreamOrArg(argv)).pipe(sms);
|
||||
if (argv['--gzip']) {
|
||||
oStream = oStream.pipe((0, zlib_1.createGzip)());
|
||||
}
|
||||
oStream.pipe(process.stdout);
|
||||
}
|
||||
else {
|
||||
const sms = new sitemap_stream_1.SitemapStream();
|
||||
if (argv['--prepend']) {
|
||||
(0, fs_1.createReadStream)(argv['--prepend'])
|
||||
.pipe(new sitemap_parser_1.XMLToSitemapItemStream())
|
||||
.pipe(sms);
|
||||
}
|
||||
const oStream = (0, utils_1.lineSeparatedURLsToSitemapOptions)(pickStreamOrArg(argv)).pipe(sms);
|
||||
if (argv['--gzip']) {
|
||||
oStream.pipe((0, zlib_1.createGzip)()).pipe(process.stdout);
|
||||
}
|
||||
else {
|
||||
oStream.pipe(process.stdout);
|
||||
}
|
||||
}
|
||||
15
node_modules/sitemap/dist/index.d.ts
generated
vendored
Normal file
15
node_modules/sitemap/dist/index.d.ts
generated
vendored
Normal file
@@ -0,0 +1,15 @@
|
||||
/*!
|
||||
* Sitemap
|
||||
* Copyright(c) 2011 Eugene Kalinin
|
||||
* MIT Licensed
|
||||
*/
|
||||
export { SitemapItemStream, SitemapItemStreamOptions, } from './lib/sitemap-item-stream';
|
||||
export { IndexTagNames, SitemapIndexStream, SitemapIndexStreamOptions, SitemapAndIndexStream, SitemapAndIndexStreamOptions, } from './lib/sitemap-index-stream';
|
||||
export { streamToPromise, SitemapStream, SitemapStreamOptions, } from './lib/sitemap-stream';
|
||||
export * from './lib/errors';
|
||||
export * from './lib/types';
|
||||
export { lineSeparatedURLsToSitemapOptions, mergeStreams, validateSMIOptions, normalizeURL, ReadlineStream, ReadlineStreamOptions, } from './lib/utils';
|
||||
export { xmlLint } from './lib/xmllint';
|
||||
export { parseSitemap, XMLToSitemapItemStream, XMLToSitemapItemStreamOptions, ObjectStreamToJSON, ObjectStreamToJSONOptions, } from './lib/sitemap-parser';
|
||||
export { parseSitemapIndex, XMLToSitemapIndexStream, XMLToSitemapIndexItemStreamOptions, IndexObjectStreamToJSON, IndexObjectStreamToJSONOptions, } from './lib/sitemap-index-parser';
|
||||
export { simpleSitemapAndIndex } from './lib/sitemap-simple';
|
||||
47
node_modules/sitemap/dist/index.js
generated
vendored
Normal file
47
node_modules/sitemap/dist/index.js
generated
vendored
Normal file
@@ -0,0 +1,47 @@
|
||||
"use strict";
|
||||
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
|
||||
}) : (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
o[k2] = m[k];
|
||||
}));
|
||||
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
||||
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.simpleSitemapAndIndex = exports.IndexObjectStreamToJSON = exports.XMLToSitemapIndexStream = exports.parseSitemapIndex = exports.ObjectStreamToJSON = exports.XMLToSitemapItemStream = exports.parseSitemap = exports.xmlLint = exports.ReadlineStream = exports.normalizeURL = exports.validateSMIOptions = exports.mergeStreams = exports.lineSeparatedURLsToSitemapOptions = exports.SitemapStream = exports.streamToPromise = exports.SitemapAndIndexStream = exports.SitemapIndexStream = exports.IndexTagNames = exports.SitemapItemStream = void 0;
|
||||
/*!
|
||||
* Sitemap
|
||||
* Copyright(c) 2011 Eugene Kalinin
|
||||
* MIT Licensed
|
||||
*/
|
||||
var sitemap_item_stream_1 = require("./lib/sitemap-item-stream");
|
||||
Object.defineProperty(exports, "SitemapItemStream", { enumerable: true, get: function () { return sitemap_item_stream_1.SitemapItemStream; } });
|
||||
var sitemap_index_stream_1 = require("./lib/sitemap-index-stream");
|
||||
Object.defineProperty(exports, "IndexTagNames", { enumerable: true, get: function () { return sitemap_index_stream_1.IndexTagNames; } });
|
||||
Object.defineProperty(exports, "SitemapIndexStream", { enumerable: true, get: function () { return sitemap_index_stream_1.SitemapIndexStream; } });
|
||||
Object.defineProperty(exports, "SitemapAndIndexStream", { enumerable: true, get: function () { return sitemap_index_stream_1.SitemapAndIndexStream; } });
|
||||
var sitemap_stream_1 = require("./lib/sitemap-stream");
|
||||
Object.defineProperty(exports, "streamToPromise", { enumerable: true, get: function () { return sitemap_stream_1.streamToPromise; } });
|
||||
Object.defineProperty(exports, "SitemapStream", { enumerable: true, get: function () { return sitemap_stream_1.SitemapStream; } });
|
||||
__exportStar(require("./lib/errors"), exports);
|
||||
__exportStar(require("./lib/types"), exports);
|
||||
var utils_1 = require("./lib/utils");
|
||||
Object.defineProperty(exports, "lineSeparatedURLsToSitemapOptions", { enumerable: true, get: function () { return utils_1.lineSeparatedURLsToSitemapOptions; } });
|
||||
Object.defineProperty(exports, "mergeStreams", { enumerable: true, get: function () { return utils_1.mergeStreams; } });
|
||||
Object.defineProperty(exports, "validateSMIOptions", { enumerable: true, get: function () { return utils_1.validateSMIOptions; } });
|
||||
Object.defineProperty(exports, "normalizeURL", { enumerable: true, get: function () { return utils_1.normalizeURL; } });
|
||||
Object.defineProperty(exports, "ReadlineStream", { enumerable: true, get: function () { return utils_1.ReadlineStream; } });
|
||||
var xmllint_1 = require("./lib/xmllint");
|
||||
Object.defineProperty(exports, "xmlLint", { enumerable: true, get: function () { return xmllint_1.xmlLint; } });
|
||||
var sitemap_parser_1 = require("./lib/sitemap-parser");
|
||||
Object.defineProperty(exports, "parseSitemap", { enumerable: true, get: function () { return sitemap_parser_1.parseSitemap; } });
|
||||
Object.defineProperty(exports, "XMLToSitemapItemStream", { enumerable: true, get: function () { return sitemap_parser_1.XMLToSitemapItemStream; } });
|
||||
Object.defineProperty(exports, "ObjectStreamToJSON", { enumerable: true, get: function () { return sitemap_parser_1.ObjectStreamToJSON; } });
|
||||
var sitemap_index_parser_1 = require("./lib/sitemap-index-parser");
|
||||
Object.defineProperty(exports, "parseSitemapIndex", { enumerable: true, get: function () { return sitemap_index_parser_1.parseSitemapIndex; } });
|
||||
Object.defineProperty(exports, "XMLToSitemapIndexStream", { enumerable: true, get: function () { return sitemap_index_parser_1.XMLToSitemapIndexStream; } });
|
||||
Object.defineProperty(exports, "IndexObjectStreamToJSON", { enumerable: true, get: function () { return sitemap_index_parser_1.IndexObjectStreamToJSON; } });
|
||||
var sitemap_simple_1 = require("./lib/sitemap-simple");
|
||||
Object.defineProperty(exports, "simpleSitemapAndIndex", { enumerable: true, get: function () { return sitemap_simple_1.simpleSitemapAndIndex; } });
|
||||
98
node_modules/sitemap/dist/lib/errors.d.ts
generated
vendored
Normal file
98
node_modules/sitemap/dist/lib/errors.d.ts
generated
vendored
Normal file
@@ -0,0 +1,98 @@
|
||||
/*!
|
||||
* Sitemap
|
||||
* Copyright(c) 2011 Eugene Kalinin
|
||||
* MIT Licensed
|
||||
*/
|
||||
/**
|
||||
* URL in SitemapItem does not exist
|
||||
*/
|
||||
export declare class NoURLError extends Error {
|
||||
constructor(message?: string);
|
||||
}
|
||||
/**
|
||||
* Config was not passed to SitemapItem constructor
|
||||
*/
|
||||
export declare class NoConfigError extends Error {
|
||||
constructor(message?: string);
|
||||
}
|
||||
/**
|
||||
* changefreq property in sitemap is invalid
|
||||
*/
|
||||
export declare class ChangeFreqInvalidError extends Error {
|
||||
constructor(url: string, changefreq: any);
|
||||
}
|
||||
/**
|
||||
* priority property in sitemap is invalid
|
||||
*/
|
||||
export declare class PriorityInvalidError extends Error {
|
||||
constructor(url: string, priority: any);
|
||||
}
|
||||
/**
|
||||
* SitemapIndex target Folder does not exists
|
||||
*/
|
||||
export declare class UndefinedTargetFolder extends Error {
|
||||
constructor(message?: string);
|
||||
}
|
||||
export declare class InvalidVideoFormat extends Error {
|
||||
constructor(url: string);
|
||||
}
|
||||
export declare class InvalidVideoDuration extends Error {
|
||||
constructor(url: string, duration: any);
|
||||
}
|
||||
export declare class InvalidVideoDescription extends Error {
|
||||
constructor(url: string, length: number);
|
||||
}
|
||||
export declare class InvalidVideoRating extends Error {
|
||||
constructor(url: string, title: any, rating: any);
|
||||
}
|
||||
export declare class InvalidAttrValue extends Error {
|
||||
constructor(key: string, val: any, validator: RegExp);
|
||||
}
|
||||
export declare class InvalidAttr extends Error {
|
||||
constructor(key: string);
|
||||
}
|
||||
export declare class InvalidNewsFormat extends Error {
|
||||
constructor(url: string);
|
||||
}
|
||||
export declare class InvalidNewsAccessValue extends Error {
|
||||
constructor(url: string, access: any);
|
||||
}
|
||||
export declare class XMLLintUnavailable extends Error {
|
||||
constructor(message?: string);
|
||||
}
|
||||
export declare class InvalidVideoTitle extends Error {
|
||||
constructor(url: string, length: number);
|
||||
}
|
||||
export declare class InvalidVideoViewCount extends Error {
|
||||
constructor(url: string, count: number);
|
||||
}
|
||||
export declare class InvalidVideoTagCount extends Error {
|
||||
constructor(url: string, count: number);
|
||||
}
|
||||
export declare class InvalidVideoCategory extends Error {
|
||||
constructor(url: string, count: number);
|
||||
}
|
||||
export declare class InvalidVideoFamilyFriendly extends Error {
|
||||
constructor(url: string, fam: string);
|
||||
}
|
||||
export declare class InvalidVideoRestriction extends Error {
|
||||
constructor(url: string, code: string);
|
||||
}
|
||||
export declare class InvalidVideoRestrictionRelationship extends Error {
|
||||
constructor(url: string, val?: string);
|
||||
}
|
||||
export declare class InvalidVideoPriceType extends Error {
|
||||
constructor(url: string, priceType?: string, price?: string);
|
||||
}
|
||||
export declare class InvalidVideoResolution extends Error {
|
||||
constructor(url: string, resolution: string);
|
||||
}
|
||||
export declare class InvalidVideoPriceCurrency extends Error {
|
||||
constructor(url: string, currency: string);
|
||||
}
|
||||
export declare class EmptyStream extends Error {
|
||||
constructor();
|
||||
}
|
||||
export declare class EmptySitemap extends Error {
|
||||
constructor();
|
||||
}
|
||||
245
node_modules/sitemap/dist/lib/errors.js
generated
vendored
Normal file
245
node_modules/sitemap/dist/lib/errors.js
generated
vendored
Normal file
@@ -0,0 +1,245 @@
|
||||
"use strict";
|
||||
/* eslint-disable @typescript-eslint/explicit-module-boundary-types */
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||
/*!
|
||||
* Sitemap
|
||||
* Copyright(c) 2011 Eugene Kalinin
|
||||
* MIT Licensed
|
||||
*/
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.EmptySitemap = exports.EmptyStream = exports.InvalidVideoPriceCurrency = exports.InvalidVideoResolution = exports.InvalidVideoPriceType = exports.InvalidVideoRestrictionRelationship = exports.InvalidVideoRestriction = exports.InvalidVideoFamilyFriendly = exports.InvalidVideoCategory = exports.InvalidVideoTagCount = exports.InvalidVideoViewCount = exports.InvalidVideoTitle = exports.XMLLintUnavailable = exports.InvalidNewsAccessValue = exports.InvalidNewsFormat = exports.InvalidAttr = exports.InvalidAttrValue = exports.InvalidVideoRating = exports.InvalidVideoDescription = exports.InvalidVideoDuration = exports.InvalidVideoFormat = exports.UndefinedTargetFolder = exports.PriorityInvalidError = exports.ChangeFreqInvalidError = exports.NoConfigError = exports.NoURLError = void 0;
|
||||
/**
|
||||
* URL in SitemapItem does not exist
|
||||
*/
|
||||
class NoURLError extends Error {
|
||||
constructor(message) {
|
||||
super(message || 'URL is required');
|
||||
this.name = 'NoURLError';
|
||||
Error.captureStackTrace(this, NoURLError);
|
||||
}
|
||||
}
|
||||
exports.NoURLError = NoURLError;
|
||||
/**
|
||||
* Config was not passed to SitemapItem constructor
|
||||
*/
|
||||
class NoConfigError extends Error {
|
||||
constructor(message) {
|
||||
super(message || 'SitemapItem requires a configuration');
|
||||
this.name = 'NoConfigError';
|
||||
Error.captureStackTrace(this, NoConfigError);
|
||||
}
|
||||
}
|
||||
exports.NoConfigError = NoConfigError;
|
||||
/**
|
||||
* changefreq property in sitemap is invalid
|
||||
*/
|
||||
class ChangeFreqInvalidError extends Error {
|
||||
constructor(url, changefreq) {
|
||||
super(`${url}: changefreq "${changefreq}" is invalid`);
|
||||
this.name = 'ChangeFreqInvalidError';
|
||||
Error.captureStackTrace(this, ChangeFreqInvalidError);
|
||||
}
|
||||
}
|
||||
exports.ChangeFreqInvalidError = ChangeFreqInvalidError;
|
||||
/**
|
||||
* priority property in sitemap is invalid
|
||||
*/
|
||||
class PriorityInvalidError extends Error {
|
||||
constructor(url, priority) {
|
||||
super(`${url}: priority "${priority}" must be a number between 0 and 1 inclusive`);
|
||||
this.name = 'PriorityInvalidError';
|
||||
Error.captureStackTrace(this, PriorityInvalidError);
|
||||
}
|
||||
}
|
||||
exports.PriorityInvalidError = PriorityInvalidError;
|
||||
/**
|
||||
* SitemapIndex target Folder does not exists
|
||||
*/
|
||||
class UndefinedTargetFolder extends Error {
|
||||
constructor(message) {
|
||||
super(message || 'Target folder must exist');
|
||||
this.name = 'UndefinedTargetFolder';
|
||||
Error.captureStackTrace(this, UndefinedTargetFolder);
|
||||
}
|
||||
}
|
||||
exports.UndefinedTargetFolder = UndefinedTargetFolder;
|
||||
class InvalidVideoFormat extends Error {
|
||||
constructor(url) {
|
||||
super(`${url} video must include thumbnail_loc, title and description fields for videos`);
|
||||
this.name = 'InvalidVideoFormat';
|
||||
Error.captureStackTrace(this, InvalidVideoFormat);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoFormat = InvalidVideoFormat;
|
||||
class InvalidVideoDuration extends Error {
|
||||
constructor(url, duration) {
|
||||
super(`${url} duration "${duration}" must be an integer of seconds between 0 and 28800`);
|
||||
this.name = 'InvalidVideoDuration';
|
||||
Error.captureStackTrace(this, InvalidVideoDuration);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoDuration = InvalidVideoDuration;
|
||||
class InvalidVideoDescription extends Error {
|
||||
constructor(url, length) {
|
||||
const message = `${url}: video description is too long ${length} vs limit of 2048 characters.`;
|
||||
super(message);
|
||||
this.name = 'InvalidVideoDescription';
|
||||
Error.captureStackTrace(this, InvalidVideoDescription);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoDescription = InvalidVideoDescription;
|
||||
class InvalidVideoRating extends Error {
|
||||
constructor(url, title, rating) {
|
||||
super(`${url}: video "${title}" rating "${rating}" must be between 0 and 5 inclusive`);
|
||||
this.name = 'InvalidVideoRating';
|
||||
Error.captureStackTrace(this, InvalidVideoRating);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoRating = InvalidVideoRating;
|
||||
class InvalidAttrValue extends Error {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
constructor(key, val, validator) {
|
||||
super('"' +
|
||||
val +
|
||||
'" tested against: ' +
|
||||
validator +
|
||||
' is not a valid value for attr: "' +
|
||||
key +
|
||||
'"');
|
||||
this.name = 'InvalidAttrValue';
|
||||
Error.captureStackTrace(this, InvalidAttrValue);
|
||||
}
|
||||
}
|
||||
exports.InvalidAttrValue = InvalidAttrValue;
|
||||
// InvalidAttr is only thrown when attrbuilder is called incorrectly internally
|
||||
/* istanbul ignore next */
|
||||
class InvalidAttr extends Error {
|
||||
constructor(key) {
|
||||
super('"' + key + '" is malformed');
|
||||
this.name = 'InvalidAttr';
|
||||
Error.captureStackTrace(this, InvalidAttr);
|
||||
}
|
||||
}
|
||||
exports.InvalidAttr = InvalidAttr;
|
||||
class InvalidNewsFormat extends Error {
|
||||
constructor(url) {
|
||||
super(`${url} News must include publication, publication name, publication language, title, and publication_date for news`);
|
||||
this.name = 'InvalidNewsFormat';
|
||||
Error.captureStackTrace(this, InvalidNewsFormat);
|
||||
}
|
||||
}
|
||||
exports.InvalidNewsFormat = InvalidNewsFormat;
|
||||
class InvalidNewsAccessValue extends Error {
|
||||
constructor(url, access) {
|
||||
super(`${url} News access "${access}" must be either Registration, Subscription or not be present`);
|
||||
this.name = 'InvalidNewsAccessValue';
|
||||
Error.captureStackTrace(this, InvalidNewsAccessValue);
|
||||
}
|
||||
}
|
||||
exports.InvalidNewsAccessValue = InvalidNewsAccessValue;
|
||||
class XMLLintUnavailable extends Error {
|
||||
constructor(message) {
|
||||
super(message || 'xmlLint is not installed. XMLLint is required to validate');
|
||||
this.name = 'XMLLintUnavailable';
|
||||
Error.captureStackTrace(this, XMLLintUnavailable);
|
||||
}
|
||||
}
|
||||
exports.XMLLintUnavailable = XMLLintUnavailable;
|
||||
class InvalidVideoTitle extends Error {
|
||||
constructor(url, length) {
|
||||
super(`${url}: video title is too long ${length} vs 100 character limit`);
|
||||
this.name = 'InvalidVideoTitle';
|
||||
Error.captureStackTrace(this, InvalidVideoTitle);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoTitle = InvalidVideoTitle;
|
||||
class InvalidVideoViewCount extends Error {
|
||||
constructor(url, count) {
|
||||
super(`${url}: video view count must be positive, view count was ${count}`);
|
||||
this.name = 'InvalidVideoViewCount';
|
||||
Error.captureStackTrace(this, InvalidVideoViewCount);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoViewCount = InvalidVideoViewCount;
|
||||
class InvalidVideoTagCount extends Error {
|
||||
constructor(url, count) {
|
||||
super(`${url}: video can have no more than 32 tags, this has ${count}`);
|
||||
this.name = 'InvalidVideoTagCount';
|
||||
Error.captureStackTrace(this, InvalidVideoTagCount);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoTagCount = InvalidVideoTagCount;
|
||||
class InvalidVideoCategory extends Error {
|
||||
constructor(url, count) {
|
||||
super(`${url}: video category can only be 256 characters but was passed ${count}`);
|
||||
this.name = 'InvalidVideoCategory';
|
||||
Error.captureStackTrace(this, InvalidVideoCategory);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoCategory = InvalidVideoCategory;
|
||||
class InvalidVideoFamilyFriendly extends Error {
|
||||
constructor(url, fam) {
|
||||
super(`${url}: video family friendly must be yes or no, was passed "${fam}"`);
|
||||
this.name = 'InvalidVideoFamilyFriendly';
|
||||
Error.captureStackTrace(this, InvalidVideoFamilyFriendly);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoFamilyFriendly = InvalidVideoFamilyFriendly;
|
||||
class InvalidVideoRestriction extends Error {
|
||||
constructor(url, code) {
|
||||
super(`${url}: video restriction must be one or more two letter country codes. Was passed "${code}"`);
|
||||
this.name = 'InvalidVideoRestriction';
|
||||
Error.captureStackTrace(this, InvalidVideoRestriction);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoRestriction = InvalidVideoRestriction;
|
||||
class InvalidVideoRestrictionRelationship extends Error {
|
||||
constructor(url, val) {
|
||||
super(`${url}: video restriction relationship must be either allow or deny. Was passed "${val}"`);
|
||||
this.name = 'InvalidVideoRestrictionRelationship';
|
||||
Error.captureStackTrace(this, InvalidVideoRestrictionRelationship);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoRestrictionRelationship = InvalidVideoRestrictionRelationship;
|
||||
class InvalidVideoPriceType extends Error {
|
||||
constructor(url, priceType, price) {
|
||||
super(priceType === undefined && price === ''
|
||||
? `${url}: video priceType is required when price is not provided`
|
||||
: `${url}: video price type "${priceType}" is not "rent" or "purchase"`);
|
||||
this.name = 'InvalidVideoPriceType';
|
||||
Error.captureStackTrace(this, InvalidVideoPriceType);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoPriceType = InvalidVideoPriceType;
|
||||
class InvalidVideoResolution extends Error {
|
||||
constructor(url, resolution) {
|
||||
super(`${url}: video price resolution "${resolution}" is not hd or sd`);
|
||||
this.name = 'InvalidVideoResolution';
|
||||
Error.captureStackTrace(this, InvalidVideoResolution);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoResolution = InvalidVideoResolution;
|
||||
class InvalidVideoPriceCurrency extends Error {
|
||||
constructor(url, currency) {
|
||||
super(`${url}: video price currency "${currency}" must be a three capital letter abbrieviation for the country currency`);
|
||||
this.name = 'InvalidVideoPriceCurrency';
|
||||
Error.captureStackTrace(this, InvalidVideoPriceCurrency);
|
||||
}
|
||||
}
|
||||
exports.InvalidVideoPriceCurrency = InvalidVideoPriceCurrency;
|
||||
class EmptyStream extends Error {
|
||||
constructor() {
|
||||
super('You have ended the stream before anything was written. streamToPromise MUST be called before ending the stream.');
|
||||
this.name = 'EmptyStream';
|
||||
Error.captureStackTrace(this, EmptyStream);
|
||||
}
|
||||
}
|
||||
exports.EmptyStream = EmptyStream;
|
||||
class EmptySitemap extends Error {
|
||||
constructor() {
|
||||
super('You ended the stream without writing anything.');
|
||||
this.name = 'EmptySitemap';
|
||||
Error.captureStackTrace(this, EmptyStream);
|
||||
}
|
||||
}
|
||||
exports.EmptySitemap = EmptySitemap;
|
||||
53
node_modules/sitemap/dist/lib/sitemap-index-parser.d.ts
generated
vendored
Normal file
53
node_modules/sitemap/dist/lib/sitemap-index-parser.d.ts
generated
vendored
Normal file
@@ -0,0 +1,53 @@
|
||||
/// <reference types="node" />
|
||||
import { SAXStream } from 'sax';
|
||||
import { Readable, Transform, TransformOptions, TransformCallback } from 'stream';
|
||||
import { IndexItem, ErrorLevel } from './types';
|
||||
declare type Logger = (level: 'warn' | 'error' | 'info' | 'log', ...message: Parameters<Console['log']>[0]) => void;
|
||||
export interface XMLToSitemapIndexItemStreamOptions extends TransformOptions {
|
||||
level?: ErrorLevel;
|
||||
logger?: Logger | false;
|
||||
}
|
||||
/**
|
||||
* Takes a stream of xml and transforms it into a stream of IndexItems
|
||||
* Use this to parse existing sitemap indices into config options compatible with this library
|
||||
*/
|
||||
export declare class XMLToSitemapIndexStream extends Transform {
|
||||
level: ErrorLevel;
|
||||
logger: Logger;
|
||||
saxStream: SAXStream;
|
||||
constructor(opts?: XMLToSitemapIndexItemStreamOptions);
|
||||
_transform(data: string, encoding: string, callback: TransformCallback): void;
|
||||
}
|
||||
/**
|
||||
Read xml and resolve with the configuration that would produce it or reject with
|
||||
an error
|
||||
```
|
||||
const { createReadStream } = require('fs')
|
||||
const { parseSitemapIndex, createSitemap } = require('sitemap')
|
||||
parseSitemapIndex(createReadStream('./example-index.xml')).then(
|
||||
// produces the same xml
|
||||
// you can, of course, more practically modify it or store it
|
||||
(xmlConfig) => console.log(createSitemap(xmlConfig).toString()),
|
||||
(err) => console.log(err)
|
||||
)
|
||||
```
|
||||
@param {Readable} xml what to parse
|
||||
@return {Promise<IndexItem[]>} resolves with list of index items that can be fed into a SitemapIndexStream. Rejects with an Error object.
|
||||
*/
|
||||
export declare function parseSitemapIndex(xml: Readable): Promise<IndexItem[]>;
|
||||
export interface IndexObjectStreamToJSONOptions extends TransformOptions {
|
||||
lineSeparated: boolean;
|
||||
}
|
||||
/**
|
||||
* A Transform that converts a stream of objects into a JSON Array or a line
|
||||
* separated stringified JSON
|
||||
* @param [lineSeparated=false] whether to separate entries by a new line or comma
|
||||
*/
|
||||
export declare class IndexObjectStreamToJSON extends Transform {
|
||||
lineSeparated: boolean;
|
||||
firstWritten: boolean;
|
||||
constructor(opts?: IndexObjectStreamToJSONOptions);
|
||||
_transform(chunk: IndexItem, encoding: string, cb: TransformCallback): void;
|
||||
_flush(cb: TransformCallback): void;
|
||||
}
|
||||
export {};
|
||||
182
node_modules/sitemap/dist/lib/sitemap-index-parser.js
generated
vendored
Normal file
182
node_modules/sitemap/dist/lib/sitemap-index-parser.js
generated
vendored
Normal file
@@ -0,0 +1,182 @@
|
||||
"use strict";
|
||||
var __importDefault = (this && this.__importDefault) || function (mod) {
|
||||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.IndexObjectStreamToJSON = exports.parseSitemapIndex = exports.XMLToSitemapIndexStream = void 0;
|
||||
const sax_1 = __importDefault(require("sax"));
|
||||
const stream_1 = require("stream");
|
||||
const types_1 = require("./types");
|
||||
function isValidTagName(tagName) {
|
||||
// This only works because the enum name and value are the same
|
||||
return tagName in types_1.IndexTagNames;
|
||||
}
|
||||
function tagTemplate() {
|
||||
return {
|
||||
url: '',
|
||||
};
|
||||
}
|
||||
const defaultLogger = (level, ...message) => console[level](...message);
|
||||
const defaultStreamOpts = {
|
||||
logger: defaultLogger,
|
||||
};
|
||||
// TODO does this need to end with `options`
|
||||
/**
|
||||
* Takes a stream of xml and transforms it into a stream of IndexItems
|
||||
* Use this to parse existing sitemap indices into config options compatible with this library
|
||||
*/
|
||||
class XMLToSitemapIndexStream extends stream_1.Transform {
|
||||
constructor(opts = defaultStreamOpts) {
|
||||
var _a;
|
||||
opts.objectMode = true;
|
||||
super(opts);
|
||||
this.saxStream = sax_1.default.createStream(true, {
|
||||
xmlns: true,
|
||||
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
||||
// @ts-ignore
|
||||
strictEntities: true,
|
||||
trim: true,
|
||||
});
|
||||
this.level = opts.level || types_1.ErrorLevel.WARN;
|
||||
if (this.level !== types_1.ErrorLevel.SILENT && opts.logger !== false) {
|
||||
this.logger = (_a = opts.logger) !== null && _a !== void 0 ? _a : defaultLogger;
|
||||
}
|
||||
else {
|
||||
this.logger = () => undefined;
|
||||
}
|
||||
let currentItem = tagTemplate();
|
||||
let currentTag;
|
||||
this.saxStream.on('opentagstart', (tag) => {
|
||||
currentTag = tag.name;
|
||||
});
|
||||
this.saxStream.on('opentag', (tag) => {
|
||||
if (!isValidTagName(tag.name)) {
|
||||
this.logger('warn', 'unhandled tag', tag.name);
|
||||
}
|
||||
});
|
||||
this.saxStream.on('text', (text) => {
|
||||
switch (currentTag) {
|
||||
case types_1.IndexTagNames.loc:
|
||||
currentItem.url = text;
|
||||
break;
|
||||
case types_1.IndexTagNames.lastmod:
|
||||
currentItem.lastmod = text;
|
||||
break;
|
||||
default:
|
||||
this.logger('log', 'unhandled text for tag:', currentTag, `'${text}'`);
|
||||
break;
|
||||
}
|
||||
});
|
||||
this.saxStream.on('cdata', (_text) => {
|
||||
switch (currentTag) {
|
||||
default:
|
||||
this.logger('log', 'unhandled cdata for tag:', currentTag);
|
||||
break;
|
||||
}
|
||||
});
|
||||
this.saxStream.on('attribute', (attr) => {
|
||||
switch (currentTag) {
|
||||
case types_1.IndexTagNames.sitemapindex:
|
||||
break;
|
||||
default:
|
||||
this.logger('log', 'unhandled attr', currentTag, attr.name);
|
||||
}
|
||||
});
|
||||
this.saxStream.on('closetag', (tag) => {
|
||||
switch (tag) {
|
||||
case types_1.IndexTagNames.sitemap:
|
||||
this.push(currentItem);
|
||||
currentItem = tagTemplate();
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
});
|
||||
}
|
||||
_transform(data, encoding, callback) {
|
||||
try {
|
||||
// correcting the type here can be done without making it a breaking change
|
||||
// TODO fix this
|
||||
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
||||
// @ts-ignore
|
||||
this.saxStream.write(data, encoding);
|
||||
callback();
|
||||
}
|
||||
catch (error) {
|
||||
callback(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
exports.XMLToSitemapIndexStream = XMLToSitemapIndexStream;
|
||||
/**
|
||||
Read xml and resolve with the configuration that would produce it or reject with
|
||||
an error
|
||||
```
|
||||
const { createReadStream } = require('fs')
|
||||
const { parseSitemapIndex, createSitemap } = require('sitemap')
|
||||
parseSitemapIndex(createReadStream('./example-index.xml')).then(
|
||||
// produces the same xml
|
||||
// you can, of course, more practically modify it or store it
|
||||
(xmlConfig) => console.log(createSitemap(xmlConfig).toString()),
|
||||
(err) => console.log(err)
|
||||
)
|
||||
```
|
||||
@param {Readable} xml what to parse
|
||||
@return {Promise<IndexItem[]>} resolves with list of index items that can be fed into a SitemapIndexStream. Rejects with an Error object.
|
||||
*/
|
||||
async function parseSitemapIndex(xml) {
|
||||
const urls = [];
|
||||
return new Promise((resolve, reject) => {
|
||||
xml
|
||||
.pipe(new XMLToSitemapIndexStream())
|
||||
.on('data', (smi) => urls.push(smi))
|
||||
.on('end', () => {
|
||||
resolve(urls);
|
||||
})
|
||||
.on('error', (error) => {
|
||||
reject(error);
|
||||
});
|
||||
});
|
||||
}
|
||||
exports.parseSitemapIndex = parseSitemapIndex;
|
||||
const defaultObjectStreamOpts = {
|
||||
lineSeparated: false,
|
||||
};
|
||||
/**
|
||||
* A Transform that converts a stream of objects into a JSON Array or a line
|
||||
* separated stringified JSON
|
||||
* @param [lineSeparated=false] whether to separate entries by a new line or comma
|
||||
*/
|
||||
class IndexObjectStreamToJSON extends stream_1.Transform {
|
||||
constructor(opts = defaultObjectStreamOpts) {
|
||||
opts.writableObjectMode = true;
|
||||
super(opts);
|
||||
this.lineSeparated = opts.lineSeparated;
|
||||
this.firstWritten = false;
|
||||
}
|
||||
_transform(chunk, encoding, cb) {
|
||||
if (!this.firstWritten) {
|
||||
this.firstWritten = true;
|
||||
if (!this.lineSeparated) {
|
||||
this.push('[');
|
||||
}
|
||||
}
|
||||
else if (this.lineSeparated) {
|
||||
this.push('\n');
|
||||
}
|
||||
else {
|
||||
this.push(',');
|
||||
}
|
||||
if (chunk) {
|
||||
this.push(JSON.stringify(chunk));
|
||||
}
|
||||
cb();
|
||||
}
|
||||
_flush(cb) {
|
||||
if (!this.lineSeparated) {
|
||||
this.push(']');
|
||||
}
|
||||
cb();
|
||||
}
|
||||
}
|
||||
exports.IndexObjectStreamToJSON = IndexObjectStreamToJSON;
|
||||
143
node_modules/sitemap/dist/lib/sitemap-index-stream.d.ts
generated
vendored
Normal file
143
node_modules/sitemap/dist/lib/sitemap-index-stream.d.ts
generated
vendored
Normal file
@@ -0,0 +1,143 @@
|
||||
/// <reference types="node" />
|
||||
import { WriteStream } from 'fs';
|
||||
import { Transform, TransformOptions, TransformCallback } from 'stream';
|
||||
import { IndexItem, SitemapItemLoose, ErrorLevel } from './types';
|
||||
import { SitemapStream } from './sitemap-stream';
|
||||
export declare enum IndexTagNames {
|
||||
sitemap = "sitemap",
|
||||
loc = "loc",
|
||||
lastmod = "lastmod"
|
||||
}
|
||||
/**
|
||||
* Options for the SitemapIndexStream
|
||||
*/
|
||||
export interface SitemapIndexStreamOptions extends TransformOptions {
|
||||
/**
|
||||
* Whether to output the lastmod date only (no time)
|
||||
*
|
||||
* @default false
|
||||
*/
|
||||
lastmodDateOnly?: boolean;
|
||||
/**
|
||||
* How to handle errors in passed in urls
|
||||
*
|
||||
* @default ErrorLevel.WARN
|
||||
*/
|
||||
level?: ErrorLevel;
|
||||
/**
|
||||
* URL to an XSL stylesheet to include in the XML
|
||||
*/
|
||||
xslUrl?: string;
|
||||
}
|
||||
/**
|
||||
* `SitemapIndexStream` is a Transform stream that takes `IndexItem`s or sitemap URL strings and outputs a stream of sitemap index XML.
|
||||
*
|
||||
* It automatically handles the XML declaration and the opening and closing tags for the sitemap index.
|
||||
*
|
||||
* ⚠️ CAUTION: This object is `readable` and must be read (e.g. piped to a file or to /dev/null)
|
||||
* before `finish` will be emitted. Failure to read the stream will result in hangs.
|
||||
*
|
||||
* @extends {Transform}
|
||||
*/
|
||||
export declare class SitemapIndexStream extends Transform {
|
||||
lastmodDateOnly: boolean;
|
||||
level: ErrorLevel;
|
||||
xslUrl?: string;
|
||||
private hasHeadOutput;
|
||||
/**
|
||||
* `SitemapIndexStream` is a Transform stream that takes `IndexItem`s or sitemap URL strings and outputs a stream of sitemap index XML.
|
||||
*
|
||||
* It automatically handles the XML declaration and the opening and closing tags for the sitemap index.
|
||||
*
|
||||
* ⚠️ CAUTION: This object is `readable` and must be read (e.g. piped to a file or to /dev/null)
|
||||
* before `finish` will be emitted. Failure to read the stream will result in hangs.
|
||||
*
|
||||
* @param {SitemapIndexStreamOptions} [opts=defaultStreamOpts] - Stream options.
|
||||
*/
|
||||
constructor(opts?: SitemapIndexStreamOptions);
|
||||
private writeHeadOutput;
|
||||
_transform(item: IndexItem | string, encoding: string, callback: TransformCallback): void;
|
||||
_flush(cb: TransformCallback): void;
|
||||
}
|
||||
declare type getSitemapStreamFunc = (i: number) => [IndexItem | string, SitemapStream, WriteStream];
|
||||
/**
|
||||
* Options for the SitemapAndIndexStream
|
||||
*
|
||||
* @extends {SitemapIndexStreamOptions}
|
||||
*/
|
||||
export interface SitemapAndIndexStreamOptions extends SitemapIndexStreamOptions {
|
||||
/**
|
||||
* Max number of items in each sitemap XML file.
|
||||
*
|
||||
* When the limit is reached the current sitemap file will be closed,
|
||||
* a wait for `finish` on the target write stream will happen,
|
||||
* and a new sitemap file will be created.
|
||||
*
|
||||
* Range: 1 - 50,000
|
||||
*
|
||||
* @default 45000
|
||||
*/
|
||||
limit?: number;
|
||||
/**
|
||||
* Callback for SitemapIndexAndStream that creates a new sitemap stream for a given sitemap index.
|
||||
*
|
||||
* Called when a new sitemap file is needed.
|
||||
*
|
||||
* The write stream is the destination where the sitemap was piped.
|
||||
* SitemapAndIndexStream will wait for the `finish` event on each sitemap's
|
||||
* write stream before moving on to the next sitemap. This ensures that the
|
||||
* contents of the write stream will be fully written before being used
|
||||
* by any following operations (e.g. uploading, reading contents for unit tests).
|
||||
*
|
||||
* @param i - The index of the sitemap file
|
||||
* @returns A tuple containing the index item to be written into the sitemap index, the sitemap stream, and the write stream for the sitemap pipe destination
|
||||
*/
|
||||
getSitemapStream: getSitemapStreamFunc;
|
||||
}
|
||||
/**
|
||||
* `SitemapAndIndexStream` is a Transform stream that takes in sitemap items,
|
||||
* writes them to sitemap files, adds the sitemap files to a sitemap index,
|
||||
* and creates new sitemap files when the count limit is reached.
|
||||
*
|
||||
* It waits for the target stream of the current sitemap file to finish before
|
||||
* moving on to the next if the target stream is returned by the `getSitemapStream`
|
||||
* callback in the 3rd position of the tuple.
|
||||
*
|
||||
* ⚠️ CAUTION: This object is `readable` and must be read (e.g. piped to a file or to /dev/null)
|
||||
* before `finish` will be emitted. Failure to read the stream will result in hangs.
|
||||
*
|
||||
* @extends {SitemapIndexStream}
|
||||
*/
|
||||
export declare class SitemapAndIndexStream extends SitemapIndexStream {
|
||||
private itemsWritten;
|
||||
private getSitemapStream;
|
||||
private currentSitemap?;
|
||||
private limit;
|
||||
private currentSitemapPipeline?;
|
||||
/**
|
||||
* `SitemapAndIndexStream` is a Transform stream that takes in sitemap items,
|
||||
* writes them to sitemap files, adds the sitemap files to a sitemap index,
|
||||
* and creates new sitemap files when the count limit is reached.
|
||||
*
|
||||
* It waits for the target stream of the current sitemap file to finish before
|
||||
* moving on to the next if the target stream is returned by the `getSitemapStream`
|
||||
* callback in the 3rd position of the tuple.
|
||||
*
|
||||
* ⚠️ CAUTION: This object is `readable` and must be read (e.g. piped to a file or to /dev/null)
|
||||
* before `finish` will be emitted. Failure to read the stream will result in hangs.
|
||||
*
|
||||
* @param {SitemapAndIndexStreamOptions} opts - Stream options.
|
||||
*/
|
||||
constructor(opts: SitemapAndIndexStreamOptions);
|
||||
_transform(item: SitemapItemLoose, encoding: string, callback: TransformCallback): void;
|
||||
private writeItem;
|
||||
/**
|
||||
* Called when the stream is finished.
|
||||
* If there is a current sitemap, we wait for it to finish before calling the callback.
|
||||
*
|
||||
* @param cb
|
||||
*/
|
||||
_flush(cb: TransformCallback): void;
|
||||
private createSitemap;
|
||||
}
|
||||
export {};
|
||||
214
node_modules/sitemap/dist/lib/sitemap-index-stream.js
generated
vendored
Normal file
214
node_modules/sitemap/dist/lib/sitemap-index-stream.js
generated
vendored
Normal file
@@ -0,0 +1,214 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.SitemapAndIndexStream = exports.SitemapIndexStream = exports.IndexTagNames = void 0;
|
||||
const stream_1 = require("stream");
|
||||
const types_1 = require("./types");
|
||||
const sitemap_stream_1 = require("./sitemap-stream");
|
||||
const sitemap_xml_1 = require("./sitemap-xml");
|
||||
var IndexTagNames;
|
||||
(function (IndexTagNames) {
|
||||
IndexTagNames["sitemap"] = "sitemap";
|
||||
IndexTagNames["loc"] = "loc";
|
||||
IndexTagNames["lastmod"] = "lastmod";
|
||||
})(IndexTagNames = exports.IndexTagNames || (exports.IndexTagNames = {}));
|
||||
const xmlDec = '<?xml version="1.0" encoding="UTF-8"?>';
|
||||
const sitemapIndexTagStart = '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
|
||||
const closetag = '</sitemapindex>';
|
||||
const defaultStreamOpts = {};
|
||||
/**
|
||||
* `SitemapIndexStream` is a Transform stream that takes `IndexItem`s or sitemap URL strings and outputs a stream of sitemap index XML.
|
||||
*
|
||||
* It automatically handles the XML declaration and the opening and closing tags for the sitemap index.
|
||||
*
|
||||
* ⚠️ CAUTION: This object is `readable` and must be read (e.g. piped to a file or to /dev/null)
|
||||
* before `finish` will be emitted. Failure to read the stream will result in hangs.
|
||||
*
|
||||
* @extends {Transform}
|
||||
*/
|
||||
class SitemapIndexStream extends stream_1.Transform {
|
||||
/**
|
||||
* `SitemapIndexStream` is a Transform stream that takes `IndexItem`s or sitemap URL strings and outputs a stream of sitemap index XML.
|
||||
*
|
||||
* It automatically handles the XML declaration and the opening and closing tags for the sitemap index.
|
||||
*
|
||||
* ⚠️ CAUTION: This object is `readable` and must be read (e.g. piped to a file or to /dev/null)
|
||||
* before `finish` will be emitted. Failure to read the stream will result in hangs.
|
||||
*
|
||||
* @param {SitemapIndexStreamOptions} [opts=defaultStreamOpts] - Stream options.
|
||||
*/
|
||||
constructor(opts = defaultStreamOpts) {
|
||||
var _a;
|
||||
opts.objectMode = true;
|
||||
super(opts);
|
||||
this.hasHeadOutput = false;
|
||||
this.lastmodDateOnly = opts.lastmodDateOnly || false;
|
||||
this.level = (_a = opts.level) !== null && _a !== void 0 ? _a : types_1.ErrorLevel.WARN;
|
||||
this.xslUrl = opts.xslUrl;
|
||||
}
|
||||
writeHeadOutput() {
|
||||
this.hasHeadOutput = true;
|
||||
let stylesheet = '';
|
||||
if (this.xslUrl) {
|
||||
stylesheet = (0, sitemap_stream_1.stylesheetInclude)(this.xslUrl);
|
||||
}
|
||||
this.push(xmlDec + stylesheet + sitemapIndexTagStart);
|
||||
}
|
||||
_transform(item, encoding, callback) {
|
||||
if (!this.hasHeadOutput) {
|
||||
this.writeHeadOutput();
|
||||
}
|
||||
this.push((0, sitemap_xml_1.otag)(IndexTagNames.sitemap));
|
||||
if (typeof item === 'string') {
|
||||
this.push((0, sitemap_xml_1.element)(IndexTagNames.loc, item));
|
||||
}
|
||||
else {
|
||||
this.push((0, sitemap_xml_1.element)(IndexTagNames.loc, item.url));
|
||||
if (item.lastmod) {
|
||||
const lastmod = new Date(item.lastmod).toISOString();
|
||||
this.push((0, sitemap_xml_1.element)(IndexTagNames.lastmod, this.lastmodDateOnly ? lastmod.slice(0, 10) : lastmod));
|
||||
}
|
||||
}
|
||||
this.push((0, sitemap_xml_1.ctag)(IndexTagNames.sitemap));
|
||||
callback();
|
||||
}
|
||||
_flush(cb) {
|
||||
if (!this.hasHeadOutput) {
|
||||
this.writeHeadOutput();
|
||||
}
|
||||
this.push(closetag);
|
||||
cb();
|
||||
}
|
||||
}
|
||||
exports.SitemapIndexStream = SitemapIndexStream;
|
||||
/**
|
||||
* `SitemapAndIndexStream` is a Transform stream that takes in sitemap items,
|
||||
* writes them to sitemap files, adds the sitemap files to a sitemap index,
|
||||
* and creates new sitemap files when the count limit is reached.
|
||||
*
|
||||
* It waits for the target stream of the current sitemap file to finish before
|
||||
* moving on to the next if the target stream is returned by the `getSitemapStream`
|
||||
* callback in the 3rd position of the tuple.
|
||||
*
|
||||
* ⚠️ CAUTION: This object is `readable` and must be read (e.g. piped to a file or to /dev/null)
|
||||
* before `finish` will be emitted. Failure to read the stream will result in hangs.
|
||||
*
|
||||
* @extends {SitemapIndexStream}
|
||||
*/
|
||||
class SitemapAndIndexStream extends SitemapIndexStream {
|
||||
/**
|
||||
* `SitemapAndIndexStream` is a Transform stream that takes in sitemap items,
|
||||
* writes them to sitemap files, adds the sitemap files to a sitemap index,
|
||||
* and creates new sitemap files when the count limit is reached.
|
||||
*
|
||||
* It waits for the target stream of the current sitemap file to finish before
|
||||
* moving on to the next if the target stream is returned by the `getSitemapStream`
|
||||
* callback in the 3rd position of the tuple.
|
||||
*
|
||||
* ⚠️ CAUTION: This object is `readable` and must be read (e.g. piped to a file or to /dev/null)
|
||||
* before `finish` will be emitted. Failure to read the stream will result in hangs.
|
||||
*
|
||||
* @param {SitemapAndIndexStreamOptions} opts - Stream options.
|
||||
*/
|
||||
constructor(opts) {
|
||||
var _a;
|
||||
opts.objectMode = true;
|
||||
super(opts);
|
||||
this.itemsWritten = 0;
|
||||
this.getSitemapStream = opts.getSitemapStream;
|
||||
this.limit = (_a = opts.limit) !== null && _a !== void 0 ? _a : 45000;
|
||||
}
|
||||
_transform(item, encoding, callback) {
|
||||
if (this.itemsWritten % this.limit === 0) {
|
||||
if (this.currentSitemap) {
|
||||
const onFinish = new Promise((resolve, reject) => {
|
||||
var _a, _b, _c;
|
||||
(_a = this.currentSitemap) === null || _a === void 0 ? void 0 : _a.on('finish', resolve);
|
||||
(_b = this.currentSitemap) === null || _b === void 0 ? void 0 : _b.on('error', reject);
|
||||
(_c = this.currentSitemap) === null || _c === void 0 ? void 0 : _c.end();
|
||||
});
|
||||
const onPipelineFinish = this.currentSitemapPipeline
|
||||
? new Promise((resolve, reject) => {
|
||||
var _a, _b;
|
||||
(_a = this.currentSitemapPipeline) === null || _a === void 0 ? void 0 : _a.on('finish', resolve);
|
||||
(_b = this.currentSitemapPipeline) === null || _b === void 0 ? void 0 : _b.on('error', reject);
|
||||
})
|
||||
: Promise.resolve();
|
||||
Promise.all([onFinish, onPipelineFinish])
|
||||
.then(() => {
|
||||
this.createSitemap(encoding);
|
||||
this.writeItem(item, callback);
|
||||
})
|
||||
.catch(callback);
|
||||
return;
|
||||
}
|
||||
else {
|
||||
this.createSitemap(encoding);
|
||||
}
|
||||
}
|
||||
this.writeItem(item, callback);
|
||||
}
|
||||
writeItem(item, callback) {
|
||||
if (!this.currentSitemap) {
|
||||
callback(new Error('No sitemap stream available'));
|
||||
return;
|
||||
}
|
||||
if (!this.currentSitemap.write(item)) {
|
||||
this.currentSitemap.once('drain', callback);
|
||||
}
|
||||
else {
|
||||
process.nextTick(callback);
|
||||
}
|
||||
// Increment the count of items written
|
||||
this.itemsWritten++;
|
||||
}
|
||||
/**
|
||||
* Called when the stream is finished.
|
||||
* If there is a current sitemap, we wait for it to finish before calling the callback.
|
||||
*
|
||||
* @param cb
|
||||
*/
|
||||
_flush(cb) {
|
||||
const onFinish = new Promise((resolve, reject) => {
|
||||
if (this.currentSitemap) {
|
||||
this.currentSitemap.on('finish', resolve);
|
||||
this.currentSitemap.on('error', reject);
|
||||
this.currentSitemap.end();
|
||||
}
|
||||
else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
const onPipelineFinish = new Promise((resolve, reject) => {
|
||||
if (this.currentSitemapPipeline) {
|
||||
this.currentSitemapPipeline.on('finish', resolve);
|
||||
this.currentSitemapPipeline.on('error', reject);
|
||||
// The pipeline (pipe target) will get it's end() call
|
||||
// from the sitemap stream ending.
|
||||
}
|
||||
else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
Promise.all([onFinish, onPipelineFinish])
|
||||
.then(() => {
|
||||
super._flush(cb);
|
||||
})
|
||||
.catch((err) => {
|
||||
cb(err);
|
||||
});
|
||||
}
|
||||
createSitemap(encoding) {
|
||||
const [idxItem, currentSitemap, currentSitemapPipeline] = this.getSitemapStream(this.itemsWritten / this.limit);
|
||||
currentSitemap.on('error', (err) => this.emit('error', err));
|
||||
this.currentSitemap = currentSitemap;
|
||||
this.currentSitemapPipeline = currentSitemapPipeline;
|
||||
super._transform(idxItem, encoding, () => {
|
||||
// We are not too fussed about waiting for the index item to be written
|
||||
// we we'll wait for the file to finish at the end
|
||||
// and index file write volume tends to be small in comprarison to sitemap
|
||||
// writes.
|
||||
// noop
|
||||
});
|
||||
}
|
||||
}
|
||||
exports.SitemapAndIndexStream = SitemapAndIndexStream;
|
||||
25
node_modules/sitemap/dist/lib/sitemap-item-stream.d.ts
generated
vendored
Normal file
25
node_modules/sitemap/dist/lib/sitemap-item-stream.d.ts
generated
vendored
Normal file
@@ -0,0 +1,25 @@
|
||||
/// <reference types="node" />
|
||||
import { Transform, TransformOptions, TransformCallback } from 'stream';
|
||||
import { SitemapItem, ErrorLevel } from './types';
|
||||
export interface StringObj {
|
||||
[index: string]: any;
|
||||
}
|
||||
export interface SitemapItemStreamOptions extends TransformOptions {
|
||||
level?: ErrorLevel;
|
||||
}
|
||||
/**
|
||||
* Takes a stream of SitemapItemOptions and spits out xml for each
|
||||
* @example
|
||||
* // writes <url><loc>https://example.com</loc><url><url><loc>https://example.com/2</loc><url>
|
||||
* const smis = new SitemapItemStream({level: 'warn'})
|
||||
* smis.pipe(writestream)
|
||||
* smis.write({url: 'https://example.com', img: [], video: [], links: []})
|
||||
* smis.write({url: 'https://example.com/2', img: [], video: [], links: []})
|
||||
* smis.end()
|
||||
* @param level - Error level
|
||||
*/
|
||||
export declare class SitemapItemStream extends Transform {
|
||||
level: ErrorLevel;
|
||||
constructor(opts?: SitemapItemStreamOptions);
|
||||
_transform(item: SitemapItem, encoding: string, callback: TransformCallback): void;
|
||||
}
|
||||
193
node_modules/sitemap/dist/lib/sitemap-item-stream.js
generated
vendored
Normal file
193
node_modules/sitemap/dist/lib/sitemap-item-stream.js
generated
vendored
Normal file
@@ -0,0 +1,193 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.SitemapItemStream = void 0;
|
||||
const stream_1 = require("stream");
|
||||
const errors_1 = require("./errors");
|
||||
const types_1 = require("./types");
|
||||
const sitemap_xml_1 = require("./sitemap-xml");
|
||||
function attrBuilder(conf, keys) {
|
||||
if (typeof keys === 'string') {
|
||||
keys = [keys];
|
||||
}
|
||||
const iv = {};
|
||||
return keys.reduce((attrs, key) => {
|
||||
// eslint-disable-next-line
|
||||
if (conf[key] !== undefined) {
|
||||
const keyAr = key.split(':');
|
||||
if (keyAr.length !== 2) {
|
||||
throw new errors_1.InvalidAttr(key);
|
||||
}
|
||||
attrs[keyAr[1]] = conf[key];
|
||||
}
|
||||
return attrs;
|
||||
}, iv);
|
||||
}
|
||||
/**
|
||||
* Takes a stream of SitemapItemOptions and spits out xml for each
|
||||
* @example
|
||||
* // writes <url><loc>https://example.com</loc><url><url><loc>https://example.com/2</loc><url>
|
||||
* const smis = new SitemapItemStream({level: 'warn'})
|
||||
* smis.pipe(writestream)
|
||||
* smis.write({url: 'https://example.com', img: [], video: [], links: []})
|
||||
* smis.write({url: 'https://example.com/2', img: [], video: [], links: []})
|
||||
* smis.end()
|
||||
* @param level - Error level
|
||||
*/
|
||||
class SitemapItemStream extends stream_1.Transform {
|
||||
constructor(opts = { level: types_1.ErrorLevel.WARN }) {
|
||||
opts.objectMode = true;
|
||||
super(opts);
|
||||
this.level = opts.level || types_1.ErrorLevel.WARN;
|
||||
}
|
||||
_transform(item, encoding, callback) {
|
||||
this.push((0, sitemap_xml_1.otag)(types_1.TagNames.url));
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames.loc, item.url));
|
||||
if (item.lastmod) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames.lastmod, item.lastmod));
|
||||
}
|
||||
if (item.changefreq) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames.changefreq, item.changefreq));
|
||||
}
|
||||
if (item.priority !== undefined && item.priority !== null) {
|
||||
if (item.fullPrecisionPriority) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames.priority, item.priority.toString()));
|
||||
}
|
||||
else {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames.priority, item.priority.toFixed(1)));
|
||||
}
|
||||
}
|
||||
item.video.forEach((video) => {
|
||||
this.push((0, sitemap_xml_1.otag)(types_1.TagNames['video:video']));
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:thumbnail_loc'], video.thumbnail_loc));
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:title'], video.title));
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:description'], video.description));
|
||||
if (video.content_loc) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:content_loc'], video.content_loc));
|
||||
}
|
||||
if (video.player_loc) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:player_loc'], attrBuilder(video, [
|
||||
'player_loc:autoplay',
|
||||
'player_loc:allow_embed',
|
||||
]), video.player_loc));
|
||||
}
|
||||
if (video.duration) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:duration'], video.duration.toString()));
|
||||
}
|
||||
if (video.expiration_date) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:expiration_date'], video.expiration_date));
|
||||
}
|
||||
if (video.rating !== undefined) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:rating'], video.rating.toString()));
|
||||
}
|
||||
if (video.view_count !== undefined) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:view_count'], video.view_count.toString()));
|
||||
}
|
||||
if (video.publication_date) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:publication_date'], video.publication_date));
|
||||
}
|
||||
for (const tag of video.tag) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:tag'], tag));
|
||||
}
|
||||
if (video.category) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:category'], video.category));
|
||||
}
|
||||
if (video.family_friendly) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:family_friendly'], video.family_friendly));
|
||||
}
|
||||
if (video.restriction) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:restriction'], attrBuilder(video, 'restriction:relationship'), video.restriction));
|
||||
}
|
||||
if (video.gallery_loc) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:gallery_loc'], { title: video['gallery_loc:title'] }, video.gallery_loc));
|
||||
}
|
||||
if (video.price) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:price'], attrBuilder(video, [
|
||||
'price:resolution',
|
||||
'price:currency',
|
||||
'price:type',
|
||||
]), video.price));
|
||||
}
|
||||
if (video.requires_subscription) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:requires_subscription'], video.requires_subscription));
|
||||
}
|
||||
if (video.uploader) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:uploader'], attrBuilder(video, 'uploader:info'), video.uploader));
|
||||
}
|
||||
if (video.platform) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:platform'], attrBuilder(video, 'platform:relationship'), video.platform));
|
||||
}
|
||||
if (video.live) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:live'], video.live));
|
||||
}
|
||||
if (video.id) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['video:id'], { type: 'url' }, video.id));
|
||||
}
|
||||
this.push((0, sitemap_xml_1.ctag)(types_1.TagNames['video:video']));
|
||||
});
|
||||
item.links.forEach((link) => {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['xhtml:link'], {
|
||||
rel: 'alternate',
|
||||
hreflang: link.lang || link.hreflang,
|
||||
href: link.url,
|
||||
}));
|
||||
});
|
||||
if (item.expires) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames.expires, new Date(item.expires).toISOString()));
|
||||
}
|
||||
if (item.androidLink) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['xhtml:link'], {
|
||||
rel: 'alternate',
|
||||
href: item.androidLink,
|
||||
}));
|
||||
}
|
||||
if (item.ampLink) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['xhtml:link'], {
|
||||
rel: 'amphtml',
|
||||
href: item.ampLink,
|
||||
}));
|
||||
}
|
||||
if (item.news) {
|
||||
this.push((0, sitemap_xml_1.otag)(types_1.TagNames['news:news']));
|
||||
this.push((0, sitemap_xml_1.otag)(types_1.TagNames['news:publication']));
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['news:name'], item.news.publication.name));
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['news:language'], item.news.publication.language));
|
||||
this.push((0, sitemap_xml_1.ctag)(types_1.TagNames['news:publication']));
|
||||
if (item.news.access) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['news:access'], item.news.access));
|
||||
}
|
||||
if (item.news.genres) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['news:genres'], item.news.genres));
|
||||
}
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['news:publication_date'], item.news.publication_date));
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['news:title'], item.news.title));
|
||||
if (item.news.keywords) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['news:keywords'], item.news.keywords));
|
||||
}
|
||||
if (item.news.stock_tickers) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['news:stock_tickers'], item.news.stock_tickers));
|
||||
}
|
||||
this.push((0, sitemap_xml_1.ctag)(types_1.TagNames['news:news']));
|
||||
}
|
||||
// Image handling
|
||||
item.img.forEach((image) => {
|
||||
this.push((0, sitemap_xml_1.otag)(types_1.TagNames['image:image']));
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['image:loc'], image.url));
|
||||
if (image.caption) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['image:caption'], image.caption));
|
||||
}
|
||||
if (image.geoLocation) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['image:geo_location'], image.geoLocation));
|
||||
}
|
||||
if (image.title) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['image:title'], image.title));
|
||||
}
|
||||
if (image.license) {
|
||||
this.push((0, sitemap_xml_1.element)(types_1.TagNames['image:license'], image.license));
|
||||
}
|
||||
this.push((0, sitemap_xml_1.ctag)(types_1.TagNames['image:image']));
|
||||
});
|
||||
this.push((0, sitemap_xml_1.ctag)(types_1.TagNames.url));
|
||||
callback();
|
||||
}
|
||||
}
|
||||
exports.SitemapItemStream = SitemapItemStream;
|
||||
55
node_modules/sitemap/dist/lib/sitemap-parser.d.ts
generated
vendored
Normal file
55
node_modules/sitemap/dist/lib/sitemap-parser.d.ts
generated
vendored
Normal file
@@ -0,0 +1,55 @@
|
||||
/// <reference types="node" />
|
||||
import { SAXStream } from 'sax';
|
||||
import { Readable, Transform, TransformOptions, TransformCallback } from 'stream';
|
||||
import { SitemapItem, ErrorLevel } from './types';
|
||||
declare type Logger = (level: 'warn' | 'error' | 'info' | 'log', ...message: Parameters<Console['log']>[0]) => void;
|
||||
export interface XMLToSitemapItemStreamOptions extends TransformOptions {
|
||||
level?: ErrorLevel;
|
||||
logger?: Logger | false;
|
||||
}
|
||||
/**
|
||||
* Takes a stream of xml and transforms it into a stream of SitemapItems
|
||||
* Use this to parse existing sitemaps into config options compatible with this library
|
||||
*/
|
||||
export declare class XMLToSitemapItemStream extends Transform {
|
||||
level: ErrorLevel;
|
||||
logger: Logger;
|
||||
error: Error | null;
|
||||
saxStream: SAXStream;
|
||||
constructor(opts?: XMLToSitemapItemStreamOptions);
|
||||
_transform(data: string, encoding: string, callback: TransformCallback): void;
|
||||
private err;
|
||||
}
|
||||
/**
|
||||
Read xml and resolve with the configuration that would produce it or reject with
|
||||
an error
|
||||
```
|
||||
const { createReadStream } = require('fs')
|
||||
const { parseSitemap, createSitemap } = require('sitemap')
|
||||
parseSitemap(createReadStream('./example.xml')).then(
|
||||
// produces the same xml
|
||||
// you can, of course, more practically modify it or store it
|
||||
(xmlConfig) => console.log(createSitemap(xmlConfig).toString()),
|
||||
(err) => console.log(err)
|
||||
)
|
||||
```
|
||||
@param {Readable} xml what to parse
|
||||
@return {Promise<SitemapItem[]>} resolves with list of sitemap items that can be fed into a SitemapStream. Rejects with an Error object.
|
||||
*/
|
||||
export declare function parseSitemap(xml: Readable): Promise<SitemapItem[]>;
|
||||
export interface ObjectStreamToJSONOptions extends TransformOptions {
|
||||
lineSeparated: boolean;
|
||||
}
|
||||
/**
|
||||
* A Transform that converts a stream of objects into a JSON Array or a line
|
||||
* separated stringified JSON
|
||||
* @param [lineSeparated=false] whether to separate entries by a new line or comma
|
||||
*/
|
||||
export declare class ObjectStreamToJSON extends Transform {
|
||||
lineSeparated: boolean;
|
||||
firstWritten: boolean;
|
||||
constructor(opts?: ObjectStreamToJSONOptions);
|
||||
_transform(chunk: SitemapItem, encoding: string, cb: TransformCallback): void;
|
||||
_flush(cb: TransformCallback): void;
|
||||
}
|
||||
export {};
|
||||
520
node_modules/sitemap/dist/lib/sitemap-parser.js
generated
vendored
Normal file
520
node_modules/sitemap/dist/lib/sitemap-parser.js
generated
vendored
Normal file
@@ -0,0 +1,520 @@
|
||||
"use strict";
|
||||
var __importDefault = (this && this.__importDefault) || function (mod) {
|
||||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.ObjectStreamToJSON = exports.parseSitemap = exports.XMLToSitemapItemStream = void 0;
|
||||
const sax_1 = __importDefault(require("sax"));
|
||||
const stream_1 = require("stream");
|
||||
const types_1 = require("./types");
|
||||
function isValidTagName(tagName) {
|
||||
// This only works because the enum name and value are the same
|
||||
return tagName in types_1.TagNames;
|
||||
}
|
||||
function tagTemplate() {
|
||||
return {
|
||||
img: [],
|
||||
video: [],
|
||||
links: [],
|
||||
url: '',
|
||||
};
|
||||
}
|
||||
function videoTemplate() {
|
||||
return {
|
||||
tag: [],
|
||||
thumbnail_loc: '',
|
||||
title: '',
|
||||
description: '',
|
||||
};
|
||||
}
|
||||
const imageTemplate = {
|
||||
url: '',
|
||||
};
|
||||
const linkTemplate = {
|
||||
lang: '',
|
||||
url: '',
|
||||
};
|
||||
function newsTemplate() {
|
||||
return {
|
||||
publication: { name: '', language: '' },
|
||||
publication_date: '',
|
||||
title: '',
|
||||
};
|
||||
}
|
||||
const defaultLogger = (level, ...message) => console[level](...message);
|
||||
const defaultStreamOpts = {
|
||||
logger: defaultLogger,
|
||||
};
|
||||
// TODO does this need to end with `options`
|
||||
/**
|
||||
* Takes a stream of xml and transforms it into a stream of SitemapItems
|
||||
* Use this to parse existing sitemaps into config options compatible with this library
|
||||
*/
|
||||
class XMLToSitemapItemStream extends stream_1.Transform {
|
||||
constructor(opts = defaultStreamOpts) {
|
||||
var _a;
|
||||
opts.objectMode = true;
|
||||
super(opts);
|
||||
this.error = null;
|
||||
this.saxStream = sax_1.default.createStream(true, {
|
||||
xmlns: true,
|
||||
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
||||
// @ts-ignore
|
||||
strictEntities: true,
|
||||
trim: true,
|
||||
});
|
||||
this.level = opts.level || types_1.ErrorLevel.WARN;
|
||||
if (this.level !== types_1.ErrorLevel.SILENT && opts.logger !== false) {
|
||||
this.logger = (_a = opts.logger) !== null && _a !== void 0 ? _a : defaultLogger;
|
||||
}
|
||||
else {
|
||||
this.logger = () => undefined;
|
||||
}
|
||||
let currentItem = tagTemplate();
|
||||
let currentTag;
|
||||
let currentVideo = videoTemplate();
|
||||
let currentImage = { ...imageTemplate };
|
||||
let currentLink = { ...linkTemplate };
|
||||
let dontpushCurrentLink = false;
|
||||
this.saxStream.on('opentagstart', (tag) => {
|
||||
currentTag = tag.name;
|
||||
if (currentTag.startsWith('news:') && !currentItem.news) {
|
||||
currentItem.news = newsTemplate();
|
||||
}
|
||||
});
|
||||
this.saxStream.on('opentag', (tag) => {
|
||||
if (isValidTagName(tag.name)) {
|
||||
if (tag.name === 'xhtml:link') {
|
||||
if (typeof tag.attributes.rel === 'string' ||
|
||||
typeof tag.attributes.href === 'string') {
|
||||
return;
|
||||
}
|
||||
if (tag.attributes.rel.value === 'alternate' &&
|
||||
tag.attributes.hreflang) {
|
||||
currentLink.url = tag.attributes.href.value;
|
||||
if (typeof tag.attributes.hreflang === 'string')
|
||||
return;
|
||||
currentLink.lang = tag.attributes.hreflang.value;
|
||||
}
|
||||
else if (tag.attributes.rel.value === 'alternate') {
|
||||
dontpushCurrentLink = true;
|
||||
currentItem.androidLink = tag.attributes.href.value;
|
||||
}
|
||||
else if (tag.attributes.rel.value === 'amphtml') {
|
||||
dontpushCurrentLink = true;
|
||||
currentItem.ampLink = tag.attributes.href.value;
|
||||
}
|
||||
else {
|
||||
this.logger('log', 'unhandled attr for xhtml:link', tag.attributes);
|
||||
this.err(`unhandled attr for xhtml:link ${tag.attributes}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
this.logger('warn', 'unhandled tag', tag.name);
|
||||
this.err(`unhandled tag: ${tag.name}`);
|
||||
}
|
||||
});
|
||||
this.saxStream.on('text', (text) => {
|
||||
switch (currentTag) {
|
||||
case 'mobile:mobile':
|
||||
break;
|
||||
case types_1.TagNames.loc:
|
||||
currentItem.url = text;
|
||||
break;
|
||||
case types_1.TagNames.changefreq:
|
||||
if ((0, types_1.isValidChangeFreq)(text)) {
|
||||
currentItem.changefreq = text;
|
||||
}
|
||||
break;
|
||||
case types_1.TagNames.priority:
|
||||
currentItem.priority = parseFloat(text);
|
||||
break;
|
||||
case types_1.TagNames.lastmod:
|
||||
currentItem.lastmod = text;
|
||||
break;
|
||||
case types_1.TagNames['video:thumbnail_loc']:
|
||||
currentVideo.thumbnail_loc = text;
|
||||
break;
|
||||
case types_1.TagNames['video:tag']:
|
||||
currentVideo.tag.push(text);
|
||||
break;
|
||||
case types_1.TagNames['video:duration']:
|
||||
currentVideo.duration = parseInt(text, 10);
|
||||
break;
|
||||
case types_1.TagNames['video:player_loc']:
|
||||
currentVideo.player_loc = text;
|
||||
break;
|
||||
case types_1.TagNames['video:content_loc']:
|
||||
currentVideo.content_loc = text;
|
||||
break;
|
||||
case types_1.TagNames['video:requires_subscription']:
|
||||
if ((0, types_1.isValidYesNo)(text)) {
|
||||
currentVideo.requires_subscription = text;
|
||||
}
|
||||
break;
|
||||
case types_1.TagNames['video:publication_date']:
|
||||
currentVideo.publication_date = text;
|
||||
break;
|
||||
case types_1.TagNames['video:id']:
|
||||
currentVideo.id = text;
|
||||
break;
|
||||
case types_1.TagNames['video:restriction']:
|
||||
currentVideo.restriction = text;
|
||||
break;
|
||||
case types_1.TagNames['video:view_count']:
|
||||
currentVideo.view_count = parseInt(text, 10);
|
||||
break;
|
||||
case types_1.TagNames['video:uploader']:
|
||||
currentVideo.uploader = text;
|
||||
break;
|
||||
case types_1.TagNames['video:family_friendly']:
|
||||
if ((0, types_1.isValidYesNo)(text)) {
|
||||
currentVideo.family_friendly = text;
|
||||
}
|
||||
break;
|
||||
case types_1.TagNames['video:expiration_date']:
|
||||
currentVideo.expiration_date = text;
|
||||
break;
|
||||
case types_1.TagNames['video:platform']:
|
||||
currentVideo.platform = text;
|
||||
break;
|
||||
case types_1.TagNames['video:price']:
|
||||
currentVideo.price = text;
|
||||
break;
|
||||
case types_1.TagNames['video:rating']:
|
||||
currentVideo.rating = parseFloat(text);
|
||||
break;
|
||||
case types_1.TagNames['video:category']:
|
||||
currentVideo.category = text;
|
||||
break;
|
||||
case types_1.TagNames['video:live']:
|
||||
if ((0, types_1.isValidYesNo)(text)) {
|
||||
currentVideo.live = text;
|
||||
}
|
||||
break;
|
||||
case types_1.TagNames['video:gallery_loc']:
|
||||
currentVideo.gallery_loc = text;
|
||||
break;
|
||||
case types_1.TagNames['image:loc']:
|
||||
currentImage.url = text;
|
||||
break;
|
||||
case types_1.TagNames['image:geo_location']:
|
||||
currentImage.geoLocation = text;
|
||||
break;
|
||||
case types_1.TagNames['image:license']:
|
||||
currentImage.license = text;
|
||||
break;
|
||||
case types_1.TagNames['news:access']:
|
||||
if (!currentItem.news) {
|
||||
currentItem.news = newsTemplate();
|
||||
}
|
||||
currentItem.news.access = text;
|
||||
break;
|
||||
case types_1.TagNames['news:genres']:
|
||||
if (!currentItem.news) {
|
||||
currentItem.news = newsTemplate();
|
||||
}
|
||||
currentItem.news.genres = text;
|
||||
break;
|
||||
case types_1.TagNames['news:publication_date']:
|
||||
if (!currentItem.news) {
|
||||
currentItem.news = newsTemplate();
|
||||
}
|
||||
currentItem.news.publication_date = text;
|
||||
break;
|
||||
case types_1.TagNames['news:keywords']:
|
||||
if (!currentItem.news) {
|
||||
currentItem.news = newsTemplate();
|
||||
}
|
||||
currentItem.news.keywords = text;
|
||||
break;
|
||||
case types_1.TagNames['news:stock_tickers']:
|
||||
if (!currentItem.news) {
|
||||
currentItem.news = newsTemplate();
|
||||
}
|
||||
currentItem.news.stock_tickers = text;
|
||||
break;
|
||||
case types_1.TagNames['news:language']:
|
||||
if (!currentItem.news) {
|
||||
currentItem.news = newsTemplate();
|
||||
}
|
||||
currentItem.news.publication.language = text;
|
||||
break;
|
||||
case types_1.TagNames['video:title']:
|
||||
currentVideo.title += text;
|
||||
break;
|
||||
case types_1.TagNames['video:description']:
|
||||
currentVideo.description += text;
|
||||
break;
|
||||
case types_1.TagNames['news:name']:
|
||||
if (!currentItem.news) {
|
||||
currentItem.news = newsTemplate();
|
||||
}
|
||||
currentItem.news.publication.name += text;
|
||||
break;
|
||||
case types_1.TagNames['news:title']:
|
||||
if (!currentItem.news) {
|
||||
currentItem.news = newsTemplate();
|
||||
}
|
||||
currentItem.news.title += text;
|
||||
break;
|
||||
case types_1.TagNames['image:caption']:
|
||||
if (!currentImage.caption) {
|
||||
currentImage.caption = text;
|
||||
}
|
||||
else {
|
||||
currentImage.caption += text;
|
||||
}
|
||||
break;
|
||||
case types_1.TagNames['image:title']:
|
||||
if (!currentImage.title) {
|
||||
currentImage.title = text;
|
||||
}
|
||||
else {
|
||||
currentImage.title += text;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
this.logger('log', 'unhandled text for tag:', currentTag, `'${text}'`);
|
||||
this.err(`unhandled text for tag: ${currentTag} '${text}'`);
|
||||
break;
|
||||
}
|
||||
});
|
||||
this.saxStream.on('cdata', (text) => {
|
||||
switch (currentTag) {
|
||||
case types_1.TagNames['video:title']:
|
||||
currentVideo.title += text;
|
||||
break;
|
||||
case types_1.TagNames['video:description']:
|
||||
currentVideo.description += text;
|
||||
break;
|
||||
case types_1.TagNames['news:name']:
|
||||
if (!currentItem.news) {
|
||||
currentItem.news = newsTemplate();
|
||||
}
|
||||
currentItem.news.publication.name += text;
|
||||
break;
|
||||
case types_1.TagNames['news:title']:
|
||||
if (!currentItem.news) {
|
||||
currentItem.news = newsTemplate();
|
||||
}
|
||||
currentItem.news.title += text;
|
||||
break;
|
||||
case types_1.TagNames['image:caption']:
|
||||
if (!currentImage.caption) {
|
||||
currentImage.caption = text;
|
||||
}
|
||||
else {
|
||||
currentImage.caption += text;
|
||||
}
|
||||
break;
|
||||
case types_1.TagNames['image:title']:
|
||||
if (!currentImage.title) {
|
||||
currentImage.title = text;
|
||||
}
|
||||
else {
|
||||
currentImage.title += text;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
this.logger('log', 'unhandled cdata for tag:', currentTag);
|
||||
this.err(`unhandled cdata for tag: ${currentTag}`);
|
||||
break;
|
||||
}
|
||||
});
|
||||
this.saxStream.on('attribute', (attr) => {
|
||||
switch (currentTag) {
|
||||
case types_1.TagNames['urlset']:
|
||||
case types_1.TagNames['xhtml:link']:
|
||||
case types_1.TagNames['video:id']:
|
||||
break;
|
||||
case types_1.TagNames['video:restriction']:
|
||||
if (attr.name === 'relationship' && (0, types_1.isAllowDeny)(attr.value)) {
|
||||
currentVideo['restriction:relationship'] = attr.value;
|
||||
}
|
||||
else {
|
||||
this.logger('log', 'unhandled attr', currentTag, attr.name);
|
||||
this.err(`unhandled attr: ${currentTag} ${attr.name}`);
|
||||
}
|
||||
break;
|
||||
case types_1.TagNames['video:price']:
|
||||
if (attr.name === 'type' && (0, types_1.isPriceType)(attr.value)) {
|
||||
currentVideo['price:type'] = attr.value;
|
||||
}
|
||||
else if (attr.name === 'currency') {
|
||||
currentVideo['price:currency'] = attr.value;
|
||||
}
|
||||
else if (attr.name === 'resolution' && (0, types_1.isResolution)(attr.value)) {
|
||||
currentVideo['price:resolution'] = attr.value;
|
||||
}
|
||||
else {
|
||||
this.logger('log', 'unhandled attr for video:price', attr.name);
|
||||
this.err(`unhandled attr: ${currentTag} ${attr.name}`);
|
||||
}
|
||||
break;
|
||||
case types_1.TagNames['video:player_loc']:
|
||||
if (attr.name === 'autoplay') {
|
||||
currentVideo['player_loc:autoplay'] = attr.value;
|
||||
}
|
||||
else if (attr.name === 'allow_embed' && (0, types_1.isValidYesNo)(attr.value)) {
|
||||
currentVideo['player_loc:allow_embed'] = attr.value;
|
||||
}
|
||||
else {
|
||||
this.logger('log', 'unhandled attr for video:player_loc', attr.name);
|
||||
this.err(`unhandled attr: ${currentTag} ${attr.name}`);
|
||||
}
|
||||
break;
|
||||
case types_1.TagNames['video:platform']:
|
||||
if (attr.name === 'relationship' && (0, types_1.isAllowDeny)(attr.value)) {
|
||||
currentVideo['platform:relationship'] = attr.value;
|
||||
}
|
||||
else {
|
||||
this.logger('log', 'unhandled attr for video:platform', attr.name, attr.value);
|
||||
this.err(`unhandled attr: ${currentTag} ${attr.name} ${attr.value}`);
|
||||
}
|
||||
break;
|
||||
case types_1.TagNames['video:gallery_loc']:
|
||||
if (attr.name === 'title') {
|
||||
currentVideo['gallery_loc:title'] = attr.value;
|
||||
}
|
||||
else {
|
||||
this.logger('log', 'unhandled attr for video:galler_loc', attr.name);
|
||||
this.err(`unhandled attr: ${currentTag} ${attr.name}`);
|
||||
}
|
||||
break;
|
||||
case types_1.TagNames['video:uploader']:
|
||||
if (attr.name === 'info') {
|
||||
currentVideo['uploader:info'] = attr.value;
|
||||
}
|
||||
else {
|
||||
this.logger('log', 'unhandled attr for video:uploader', attr.name);
|
||||
this.err(`unhandled attr: ${currentTag} ${attr.name}`);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
this.logger('log', 'unhandled attr', currentTag, attr.name);
|
||||
this.err(`unhandled attr: ${currentTag} ${attr.name}`);
|
||||
}
|
||||
});
|
||||
this.saxStream.on('closetag', (tag) => {
|
||||
switch (tag) {
|
||||
case types_1.TagNames.url:
|
||||
this.push(currentItem);
|
||||
currentItem = tagTemplate();
|
||||
break;
|
||||
case types_1.TagNames['video:video']:
|
||||
currentItem.video.push(currentVideo);
|
||||
currentVideo = videoTemplate();
|
||||
break;
|
||||
case types_1.TagNames['image:image']:
|
||||
currentItem.img.push(currentImage);
|
||||
currentImage = { ...imageTemplate };
|
||||
break;
|
||||
case types_1.TagNames['xhtml:link']:
|
||||
if (!dontpushCurrentLink) {
|
||||
currentItem.links.push(currentLink);
|
||||
}
|
||||
currentLink = { ...linkTemplate };
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
});
|
||||
}
|
||||
_transform(data, encoding, callback) {
|
||||
try {
|
||||
const cb = () => callback(this.level === types_1.ErrorLevel.THROW ? this.error : null);
|
||||
// correcting the type here can be done without making it a breaking change
|
||||
// TODO fix this
|
||||
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
||||
// @ts-ignore
|
||||
if (!this.saxStream.write(data, encoding)) {
|
||||
this.saxStream.once('drain', cb);
|
||||
}
|
||||
else {
|
||||
process.nextTick(cb);
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
callback(error);
|
||||
}
|
||||
}
|
||||
err(msg) {
|
||||
if (!this.error)
|
||||
this.error = new Error(msg);
|
||||
}
|
||||
}
|
||||
exports.XMLToSitemapItemStream = XMLToSitemapItemStream;
|
||||
/**
|
||||
Read xml and resolve with the configuration that would produce it or reject with
|
||||
an error
|
||||
```
|
||||
const { createReadStream } = require('fs')
|
||||
const { parseSitemap, createSitemap } = require('sitemap')
|
||||
parseSitemap(createReadStream('./example.xml')).then(
|
||||
// produces the same xml
|
||||
// you can, of course, more practically modify it or store it
|
||||
(xmlConfig) => console.log(createSitemap(xmlConfig).toString()),
|
||||
(err) => console.log(err)
|
||||
)
|
||||
```
|
||||
@param {Readable} xml what to parse
|
||||
@return {Promise<SitemapItem[]>} resolves with list of sitemap items that can be fed into a SitemapStream. Rejects with an Error object.
|
||||
*/
|
||||
async function parseSitemap(xml) {
|
||||
const urls = [];
|
||||
return new Promise((resolve, reject) => {
|
||||
xml
|
||||
.pipe(new XMLToSitemapItemStream())
|
||||
.on('data', (smi) => urls.push(smi))
|
||||
.on('end', () => {
|
||||
resolve(urls);
|
||||
})
|
||||
.on('error', (error) => {
|
||||
reject(error);
|
||||
});
|
||||
});
|
||||
}
|
||||
exports.parseSitemap = parseSitemap;
|
||||
const defaultObjectStreamOpts = {
|
||||
lineSeparated: false,
|
||||
};
|
||||
/**
|
||||
* A Transform that converts a stream of objects into a JSON Array or a line
|
||||
* separated stringified JSON
|
||||
* @param [lineSeparated=false] whether to separate entries by a new line or comma
|
||||
*/
|
||||
class ObjectStreamToJSON extends stream_1.Transform {
|
||||
constructor(opts = defaultObjectStreamOpts) {
|
||||
opts.writableObjectMode = true;
|
||||
super(opts);
|
||||
this.lineSeparated = opts.lineSeparated;
|
||||
this.firstWritten = false;
|
||||
}
|
||||
_transform(chunk, encoding, cb) {
|
||||
if (!this.firstWritten) {
|
||||
this.firstWritten = true;
|
||||
if (!this.lineSeparated) {
|
||||
this.push('[');
|
||||
}
|
||||
}
|
||||
else if (this.lineSeparated) {
|
||||
this.push('\n');
|
||||
}
|
||||
else {
|
||||
this.push(',');
|
||||
}
|
||||
if (chunk) {
|
||||
this.push(JSON.stringify(chunk));
|
||||
}
|
||||
cb();
|
||||
}
|
||||
_flush(cb) {
|
||||
if (!this.lineSeparated) {
|
||||
this.push(']');
|
||||
}
|
||||
cb();
|
||||
}
|
||||
}
|
||||
exports.ObjectStreamToJSON = ObjectStreamToJSON;
|
||||
24
node_modules/sitemap/dist/lib/sitemap-simple.d.ts
generated
vendored
Normal file
24
node_modules/sitemap/dist/lib/sitemap-simple.d.ts
generated
vendored
Normal file
@@ -0,0 +1,24 @@
|
||||
import { Readable } from 'stream';
|
||||
import { SitemapItemLoose } from './types';
|
||||
/**
|
||||
*
|
||||
* @param {object} options -
|
||||
* @param {string} options.hostname - The hostname for all URLs
|
||||
* @param {string} [options.sitemapHostname] - The hostname for the sitemaps if different than hostname
|
||||
* @param {SitemapItemLoose[] | string | Readable | string[]} options.sourceData - The urls you want to make a sitemap out of.
|
||||
* @param {string} options.destinationDir - where to write the sitemaps and index
|
||||
* @param {string} [options.publicBasePath] - where the sitemaps are relative to the hostname. Defaults to root.
|
||||
* @param {number} [options.limit] - how many URLs to write before switching to a new file. Defaults to 50k
|
||||
* @param {boolean} [options.gzip] - whether to compress the written files. Defaults to true
|
||||
* @returns {Promise<void>} an empty promise that resolves when everything is done
|
||||
*/
|
||||
export declare const simpleSitemapAndIndex: ({ hostname, sitemapHostname, sourceData, destinationDir, limit, gzip, publicBasePath, }: {
|
||||
hostname: string;
|
||||
sitemapHostname?: string | undefined;
|
||||
sourceData: SitemapItemLoose[] | string | Readable | string[];
|
||||
destinationDir: string;
|
||||
publicBasePath?: string | undefined;
|
||||
limit?: number | undefined;
|
||||
gzip?: boolean | undefined;
|
||||
}) => Promise<void>;
|
||||
export default simpleSitemapAndIndex;
|
||||
82
node_modules/sitemap/dist/lib/sitemap-simple.js
generated
vendored
Normal file
82
node_modules/sitemap/dist/lib/sitemap-simple.js
generated
vendored
Normal file
@@ -0,0 +1,82 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.simpleSitemapAndIndex = void 0;
|
||||
const sitemap_index_stream_1 = require("./sitemap-index-stream");
|
||||
const sitemap_stream_1 = require("./sitemap-stream");
|
||||
const utils_1 = require("./utils");
|
||||
const zlib_1 = require("zlib");
|
||||
const fs_1 = require("fs");
|
||||
const path_1 = require("path");
|
||||
const stream_1 = require("stream");
|
||||
const util_1 = require("util");
|
||||
const url_1 = require("url");
|
||||
const pipeline = (0, util_1.promisify)(stream_1.pipeline);
|
||||
/**
|
||||
*
|
||||
* @param {object} options -
|
||||
* @param {string} options.hostname - The hostname for all URLs
|
||||
* @param {string} [options.sitemapHostname] - The hostname for the sitemaps if different than hostname
|
||||
* @param {SitemapItemLoose[] | string | Readable | string[]} options.sourceData - The urls you want to make a sitemap out of.
|
||||
* @param {string} options.destinationDir - where to write the sitemaps and index
|
||||
* @param {string} [options.publicBasePath] - where the sitemaps are relative to the hostname. Defaults to root.
|
||||
* @param {number} [options.limit] - how many URLs to write before switching to a new file. Defaults to 50k
|
||||
* @param {boolean} [options.gzip] - whether to compress the written files. Defaults to true
|
||||
* @returns {Promise<void>} an empty promise that resolves when everything is done
|
||||
*/
|
||||
const simpleSitemapAndIndex = async ({ hostname, sitemapHostname = hostname, // if different
|
||||
/**
|
||||
* Pass a line separated list of sitemap items or a stream or an array
|
||||
*/
|
||||
sourceData, destinationDir, limit = 50000, gzip = true, publicBasePath = './', }) => {
|
||||
await fs_1.promises.mkdir(destinationDir, { recursive: true });
|
||||
const sitemapAndIndexStream = new sitemap_index_stream_1.SitemapAndIndexStream({
|
||||
limit,
|
||||
getSitemapStream: (i) => {
|
||||
const sitemapStream = new sitemap_stream_1.SitemapStream({
|
||||
hostname,
|
||||
});
|
||||
const path = `./sitemap-${i}.xml`;
|
||||
const writePath = (0, path_1.resolve)(destinationDir, path + (gzip ? '.gz' : ''));
|
||||
if (!publicBasePath.endsWith('/')) {
|
||||
publicBasePath += '/';
|
||||
}
|
||||
const publicPath = (0, path_1.normalize)(publicBasePath + path);
|
||||
let pipeline;
|
||||
if (gzip) {
|
||||
pipeline = sitemapStream
|
||||
.pipe((0, zlib_1.createGzip)()) // compress the output of the sitemap
|
||||
.pipe((0, fs_1.createWriteStream)(writePath)); // write it to sitemap-NUMBER.xml
|
||||
}
|
||||
else {
|
||||
pipeline = sitemapStream.pipe((0, fs_1.createWriteStream)(writePath)); // write it to sitemap-NUMBER.xml
|
||||
}
|
||||
return [
|
||||
new url_1.URL(`${publicPath}${gzip ? '.gz' : ''}`, sitemapHostname).toString(),
|
||||
sitemapStream,
|
||||
pipeline,
|
||||
];
|
||||
},
|
||||
});
|
||||
let src;
|
||||
if (typeof sourceData === 'string') {
|
||||
src = (0, utils_1.lineSeparatedURLsToSitemapOptions)((0, fs_1.createReadStream)(sourceData));
|
||||
}
|
||||
else if (sourceData instanceof stream_1.Readable) {
|
||||
src = sourceData;
|
||||
}
|
||||
else if (Array.isArray(sourceData)) {
|
||||
src = stream_1.Readable.from(sourceData);
|
||||
}
|
||||
else {
|
||||
throw new Error("unhandled source type. You've passed in data that is not supported");
|
||||
}
|
||||
const writePath = (0, path_1.resolve)(destinationDir, `./sitemap-index.xml${gzip ? '.gz' : ''}`);
|
||||
if (gzip) {
|
||||
return pipeline(src, sitemapAndIndexStream, (0, zlib_1.createGzip)(), (0, fs_1.createWriteStream)(writePath));
|
||||
}
|
||||
else {
|
||||
return pipeline(src, sitemapAndIndexStream, (0, fs_1.createWriteStream)(writePath));
|
||||
}
|
||||
};
|
||||
exports.simpleSitemapAndIndex = simpleSitemapAndIndex;
|
||||
exports.default = exports.simpleSitemapAndIndex;
|
||||
52
node_modules/sitemap/dist/lib/sitemap-stream.d.ts
generated
vendored
Normal file
52
node_modules/sitemap/dist/lib/sitemap-stream.d.ts
generated
vendored
Normal file
@@ -0,0 +1,52 @@
|
||||
/// <reference types="node" />
|
||||
import { Transform, TransformOptions, TransformCallback, Readable } from 'stream';
|
||||
import { SitemapItemLoose, ErrorLevel, ErrorHandler } from './types';
|
||||
export declare const stylesheetInclude: (url: string) => string;
|
||||
export interface NSArgs {
|
||||
news: boolean;
|
||||
video: boolean;
|
||||
xhtml: boolean;
|
||||
image: boolean;
|
||||
custom?: string[];
|
||||
}
|
||||
export declare const closetag = "</urlset>";
|
||||
export interface SitemapStreamOptions extends TransformOptions {
|
||||
hostname?: string;
|
||||
level?: ErrorLevel;
|
||||
lastmodDateOnly?: boolean;
|
||||
xmlns?: NSArgs;
|
||||
xslUrl?: string;
|
||||
errorHandler?: ErrorHandler;
|
||||
}
|
||||
/**
|
||||
* A [Transform](https://nodejs.org/api/stream.html#stream_implementing_a_transform_stream)
|
||||
* for turning a
|
||||
* [Readable stream](https://nodejs.org/api/stream.html#stream_readable_streams)
|
||||
* of either [SitemapItemOptions](#sitemap-item-options) or url strings into a
|
||||
* Sitemap. The readable stream it transforms **must** be in object mode.
|
||||
*/
|
||||
export declare class SitemapStream extends Transform {
|
||||
hostname?: string;
|
||||
level: ErrorLevel;
|
||||
hasHeadOutput: boolean;
|
||||
xmlNS: NSArgs;
|
||||
xslUrl?: string;
|
||||
errorHandler?: ErrorHandler;
|
||||
private smiStream;
|
||||
lastmodDateOnly: boolean;
|
||||
constructor(opts?: SitemapStreamOptions);
|
||||
_transform(item: SitemapItemLoose, encoding: string, callback: TransformCallback): void;
|
||||
_flush(cb: TransformCallback): void;
|
||||
}
|
||||
/**
|
||||
* Converts a readable stream into a promise that resolves with the concatenated data from the stream.
|
||||
*
|
||||
* The function listens for 'data' events from the stream, and when the stream ends, it resolves the promise with the concatenated data. If an error occurs while reading from the stream, the promise is rejected with the error.
|
||||
*
|
||||
* ⚠️ CAUTION: This function should not generally be used in production / when writing to files as it holds a copy of the entire file contents in memory until finished.
|
||||
*
|
||||
* @param {Readable} stream - The readable stream to convert to a promise.
|
||||
* @returns {Promise<Buffer>} A promise that resolves with the concatenated data from the stream as a Buffer, or rejects with an error if one occurred while reading from the stream. If the stream is empty, the promise is rejected with an EmptyStream error.
|
||||
* @throws {EmptyStream} If the stream is empty.
|
||||
*/
|
||||
export declare function streamToPromise(stream: Readable): Promise<Buffer>;
|
||||
129
node_modules/sitemap/dist/lib/sitemap-stream.js
generated
vendored
Normal file
129
node_modules/sitemap/dist/lib/sitemap-stream.js
generated
vendored
Normal file
@@ -0,0 +1,129 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.streamToPromise = exports.SitemapStream = exports.closetag = exports.stylesheetInclude = void 0;
|
||||
const stream_1 = require("stream");
|
||||
const types_1 = require("./types");
|
||||
const utils_1 = require("./utils");
|
||||
const sitemap_item_stream_1 = require("./sitemap-item-stream");
|
||||
const errors_1 = require("./errors");
|
||||
const xmlDec = '<?xml version="1.0" encoding="UTF-8"?>';
|
||||
const stylesheetInclude = (url) => {
|
||||
return `<?xml-stylesheet type="text/xsl" href="${url}"?>`;
|
||||
};
|
||||
exports.stylesheetInclude = stylesheetInclude;
|
||||
const urlsetTagStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"';
|
||||
const getURLSetNs = ({ news, video, image, xhtml, custom }, xslURL) => {
|
||||
let ns = xmlDec;
|
||||
if (xslURL) {
|
||||
ns += (0, exports.stylesheetInclude)(xslURL);
|
||||
}
|
||||
ns += urlsetTagStart;
|
||||
if (news) {
|
||||
ns += ' xmlns:news="http://www.google.com/schemas/sitemap-news/0.9"';
|
||||
}
|
||||
if (xhtml) {
|
||||
ns += ' xmlns:xhtml="http://www.w3.org/1999/xhtml"';
|
||||
}
|
||||
if (image) {
|
||||
ns += ' xmlns:image="http://www.google.com/schemas/sitemap-image/1.1"';
|
||||
}
|
||||
if (video) {
|
||||
ns += ' xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"';
|
||||
}
|
||||
if (custom) {
|
||||
ns += ' ' + custom.join(' ');
|
||||
}
|
||||
return ns + '>';
|
||||
};
|
||||
exports.closetag = '</urlset>';
|
||||
const defaultXMLNS = {
|
||||
news: true,
|
||||
xhtml: true,
|
||||
image: true,
|
||||
video: true,
|
||||
};
|
||||
const defaultStreamOpts = {
|
||||
xmlns: defaultXMLNS,
|
||||
};
|
||||
/**
|
||||
* A [Transform](https://nodejs.org/api/stream.html#stream_implementing_a_transform_stream)
|
||||
* for turning a
|
||||
* [Readable stream](https://nodejs.org/api/stream.html#stream_readable_streams)
|
||||
* of either [SitemapItemOptions](#sitemap-item-options) or url strings into a
|
||||
* Sitemap. The readable stream it transforms **must** be in object mode.
|
||||
*/
|
||||
class SitemapStream extends stream_1.Transform {
|
||||
constructor(opts = defaultStreamOpts) {
|
||||
opts.objectMode = true;
|
||||
super(opts);
|
||||
this.hasHeadOutput = false;
|
||||
this.hostname = opts.hostname;
|
||||
this.level = opts.level || types_1.ErrorLevel.WARN;
|
||||
this.errorHandler = opts.errorHandler;
|
||||
this.smiStream = new sitemap_item_stream_1.SitemapItemStream({ level: opts.level });
|
||||
this.smiStream.on('data', (data) => this.push(data));
|
||||
this.lastmodDateOnly = opts.lastmodDateOnly || false;
|
||||
this.xmlNS = opts.xmlns || defaultXMLNS;
|
||||
this.xslUrl = opts.xslUrl;
|
||||
}
|
||||
_transform(item, encoding, callback) {
|
||||
if (!this.hasHeadOutput) {
|
||||
this.hasHeadOutput = true;
|
||||
this.push(getURLSetNs(this.xmlNS, this.xslUrl));
|
||||
}
|
||||
if (!this.smiStream.write((0, utils_1.validateSMIOptions)((0, utils_1.normalizeURL)(item, this.hostname, this.lastmodDateOnly), this.level, this.errorHandler))) {
|
||||
this.smiStream.once('drain', callback);
|
||||
}
|
||||
else {
|
||||
process.nextTick(callback);
|
||||
}
|
||||
}
|
||||
_flush(cb) {
|
||||
if (!this.hasHeadOutput) {
|
||||
cb(new errors_1.EmptySitemap());
|
||||
}
|
||||
else {
|
||||
this.push(exports.closetag);
|
||||
cb();
|
||||
}
|
||||
}
|
||||
}
|
||||
exports.SitemapStream = SitemapStream;
|
||||
/**
|
||||
* Converts a readable stream into a promise that resolves with the concatenated data from the stream.
|
||||
*
|
||||
* The function listens for 'data' events from the stream, and when the stream ends, it resolves the promise with the concatenated data. If an error occurs while reading from the stream, the promise is rejected with the error.
|
||||
*
|
||||
* ⚠️ CAUTION: This function should not generally be used in production / when writing to files as it holds a copy of the entire file contents in memory until finished.
|
||||
*
|
||||
* @param {Readable} stream - The readable stream to convert to a promise.
|
||||
* @returns {Promise<Buffer>} A promise that resolves with the concatenated data from the stream as a Buffer, or rejects with an error if one occurred while reading from the stream. If the stream is empty, the promise is rejected with an EmptyStream error.
|
||||
* @throws {EmptyStream} If the stream is empty.
|
||||
*/
|
||||
function streamToPromise(stream) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const drain = [];
|
||||
stream
|
||||
// Error propagation is not automatic
|
||||
// Bubble up errors on the read stream
|
||||
.on('error', reject)
|
||||
.pipe(new stream_1.Writable({
|
||||
write(chunk, enc, next) {
|
||||
drain.push(chunk);
|
||||
next();
|
||||
},
|
||||
}))
|
||||
// This bubbles up errors when writing to the internal buffer
|
||||
// This is unlikely to happen, but we have this for completeness
|
||||
.on('error', reject)
|
||||
.on('finish', () => {
|
||||
if (!drain.length) {
|
||||
reject(new errors_1.EmptyStream());
|
||||
}
|
||||
else {
|
||||
resolve(Buffer.concat(drain));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
exports.streamToPromise = streamToPromise;
|
||||
9
node_modules/sitemap/dist/lib/sitemap-xml.d.ts
generated
vendored
Normal file
9
node_modules/sitemap/dist/lib/sitemap-xml.d.ts
generated
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
import { TagNames } from './types';
|
||||
import { StringObj } from './sitemap-item-stream';
|
||||
import { IndexTagNames } from './sitemap-index-stream';
|
||||
export declare function text(txt: string): string;
|
||||
export declare function otag(nodeName: TagNames | IndexTagNames, attrs?: StringObj, selfClose?: boolean): string;
|
||||
export declare function ctag(nodeName: TagNames | IndexTagNames): string;
|
||||
export declare function element(nodeName: TagNames, attrs: StringObj, innerText: string): string;
|
||||
export declare function element(nodeName: TagNames | IndexTagNames, innerText: string): string;
|
||||
export declare function element(nodeName: TagNames, attrs: StringObj): string;
|
||||
47
node_modules/sitemap/dist/lib/sitemap-xml.js
generated
vendored
Normal file
47
node_modules/sitemap/dist/lib/sitemap-xml.js
generated
vendored
Normal file
@@ -0,0 +1,47 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.element = exports.ctag = exports.otag = exports.text = void 0;
|
||||
const invalidXMLUnicodeRegex =
|
||||
// eslint-disable-next-line no-control-regex
|
||||
/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u0084\u0086-\u009F\uD800-\uDFFF\uFDD0-\uFDDF\u{1FFFE}-\u{1FFFF}\u{2FFFE}-\u{2FFFF}\u{3FFFE}-\u{3FFFF}\u{4FFFE}-\u{4FFFF}\u{5FFFE}-\u{5FFFF}\u{6FFFE}-\u{6FFFF}\u{7FFFE}-\u{7FFFF}\u{8FFFE}-\u{8FFFF}\u{9FFFE}-\u{9FFFF}\u{AFFFE}-\u{AFFFF}\u{BFFFE}-\u{BFFFF}\u{CFFFE}-\u{CFFFF}\u{DFFFE}-\u{DFFFF}\u{EFFFE}-\u{EFFFF}\u{FFFFE}-\u{FFFFF}\u{10FFFE}-\u{10FFFF}]/gu;
|
||||
const amp = /&/g;
|
||||
const lt = /</g;
|
||||
const apos = /'/g;
|
||||
const quot = /"/g;
|
||||
function text(txt) {
|
||||
return txt
|
||||
.replace(amp, '&')
|
||||
.replace(lt, '<')
|
||||
.replace(invalidXMLUnicodeRegex, '');
|
||||
}
|
||||
exports.text = text;
|
||||
function otag(nodeName, attrs, selfClose = false) {
|
||||
let attrstr = '';
|
||||
for (const k in attrs) {
|
||||
const val = attrs[k]
|
||||
.replace(amp, '&')
|
||||
.replace(lt, '<')
|
||||
.replace(apos, ''')
|
||||
.replace(quot, '"')
|
||||
.replace(invalidXMLUnicodeRegex, '');
|
||||
attrstr += ` ${k}="${val}"`;
|
||||
}
|
||||
return `<${nodeName}${attrstr}${selfClose ? '/' : ''}>`;
|
||||
}
|
||||
exports.otag = otag;
|
||||
function ctag(nodeName) {
|
||||
return `</${nodeName}>`;
|
||||
}
|
||||
exports.ctag = ctag;
|
||||
function element(nodeName, attrs, innerText) {
|
||||
if (typeof attrs === 'string') {
|
||||
return otag(nodeName) + text(attrs) + ctag(nodeName);
|
||||
}
|
||||
else if (innerText) {
|
||||
return otag(nodeName, attrs) + text(innerText) + ctag(nodeName);
|
||||
}
|
||||
else {
|
||||
return otag(nodeName, attrs, true);
|
||||
}
|
||||
}
|
||||
exports.element = element;
|
||||
393
node_modules/sitemap/dist/lib/types.d.ts
generated
vendored
Normal file
393
node_modules/sitemap/dist/lib/types.d.ts
generated
vendored
Normal file
@@ -0,0 +1,393 @@
|
||||
/// <reference types="node" />
|
||||
import { URL } from 'url';
|
||||
/**
|
||||
* How frequently the page is likely to change. This value provides general
|
||||
* information to search engines and may not correlate exactly to how often they crawl the page. Please note that the
|
||||
* value of this tag is considered a hint and not a command. See
|
||||
* <https://www.sitemaps.org/protocol.html#xmlTagDefinitions> for the acceptable
|
||||
* values
|
||||
*/
|
||||
export declare enum EnumChangefreq {
|
||||
DAILY = "daily",
|
||||
MONTHLY = "monthly",
|
||||
ALWAYS = "always",
|
||||
HOURLY = "hourly",
|
||||
WEEKLY = "weekly",
|
||||
YEARLY = "yearly",
|
||||
NEVER = "never"
|
||||
}
|
||||
export declare const validators: {
|
||||
[index: string]: RegExp;
|
||||
};
|
||||
export declare function isPriceType(pt: string | PriceType): pt is PriceType;
|
||||
export declare function isResolution(res: string): res is Resolution;
|
||||
export declare const CHANGEFREQ: EnumChangefreq[];
|
||||
export declare function isValidChangeFreq(freq: string): freq is EnumChangefreq;
|
||||
export declare enum EnumYesNo {
|
||||
YES = "YES",
|
||||
NO = "NO",
|
||||
Yes = "Yes",
|
||||
No = "No",
|
||||
yes = "yes",
|
||||
no = "no"
|
||||
}
|
||||
export declare function isValidYesNo(yn: string): yn is EnumYesNo;
|
||||
export declare enum EnumAllowDeny {
|
||||
ALLOW = "allow",
|
||||
DENY = "deny"
|
||||
}
|
||||
export declare function isAllowDeny(ad: string): ad is EnumAllowDeny;
|
||||
/**
|
||||
* https://support.google.com/webmasters/answer/74288?hl=en&ref_topic=4581190
|
||||
*/
|
||||
export interface NewsItem {
|
||||
access?: 'Registration' | 'Subscription';
|
||||
publication: {
|
||||
name: string;
|
||||
/**
|
||||
* The `<language>` is the language of your publication. Use an ISO 639
|
||||
* language code (2 or 3 letters).
|
||||
*/
|
||||
language: string;
|
||||
};
|
||||
/**
|
||||
* @example 'PressRelease, Blog'
|
||||
*/
|
||||
genres?: string;
|
||||
/**
|
||||
* Article publication date in W3C format, using either the "complete date" (YYYY-MM-DD) format or the "complete date
|
||||
* plus hours, minutes, and seconds"
|
||||
*/
|
||||
publication_date: string;
|
||||
/**
|
||||
* The title of the news article
|
||||
* @example 'Companies A, B in Merger Talks'
|
||||
*/
|
||||
title: string;
|
||||
/**
|
||||
* @example 'business, merger, acquisition'
|
||||
*/
|
||||
keywords?: string;
|
||||
/**
|
||||
* @example 'NASDAQ:A, NASDAQ:B'
|
||||
*/
|
||||
stock_tickers?: string;
|
||||
}
|
||||
/**
|
||||
* Sitemap Image
|
||||
* https://support.google.com/webmasters/answer/178636?hl=en&ref_topic=4581190
|
||||
*/
|
||||
export interface Img {
|
||||
/**
|
||||
* The URL of the image
|
||||
* @example 'https://example.com/image.jpg'
|
||||
*/
|
||||
url: string;
|
||||
/**
|
||||
* The caption of the image
|
||||
* @example 'Thanksgiving dinner'
|
||||
*/
|
||||
caption?: string;
|
||||
/**
|
||||
* The title of the image
|
||||
* @example 'Star Wars EP IV'
|
||||
*/
|
||||
title?: string;
|
||||
/**
|
||||
* The geographic location of the image.
|
||||
* @example 'Limerick, Ireland'
|
||||
*/
|
||||
geoLocation?: string;
|
||||
/**
|
||||
* A URL to the license of the image.
|
||||
* @example 'https://example.com/license.txt'
|
||||
*/
|
||||
license?: string;
|
||||
}
|
||||
interface VideoItemBase {
|
||||
/**
|
||||
* A URL pointing to the video thumbnail image file
|
||||
* @example "https://rtv3-img-roosterteeth.akamaized.net/store/0e841100-289b-4184-ae30-b6a16736960a.jpg/sm/thumb3.jpg"
|
||||
*/
|
||||
thumbnail_loc: string;
|
||||
/**
|
||||
* The title of the video
|
||||
* @example '2018:E6 - GoldenEye: Source'
|
||||
*/
|
||||
title: string;
|
||||
/**
|
||||
* A description of the video. Maximum 2048 characters.
|
||||
* @example 'We play gun game in GoldenEye: Source with a good friend of ours. His name is Gruchy. Dan Gruchy.'
|
||||
*/
|
||||
description: string;
|
||||
/**
|
||||
* A URL pointing to the actual video media file. Should be one of the supported formats. HTML is not a supported
|
||||
* format. Flash is allowed, but no longer supported on most mobile platforms, and so may be indexed less well. Must
|
||||
* not be the same as the `<loc>` URL.
|
||||
* @example "http://streamserver.example.com/video123.mp4"
|
||||
*/
|
||||
content_loc?: string;
|
||||
/**
|
||||
* A URL pointing to a player for a specific video. Usually this is the information in the src element of an `<embed>`
|
||||
* tag. Must not be the same as the `<loc>` URL
|
||||
* @example "https://roosterteeth.com/embed/rouletsplay-2018-goldeneye-source"
|
||||
*/
|
||||
player_loc?: string;
|
||||
/**
|
||||
* A string the search engine can append as a query param to enable automatic
|
||||
* playback. Equivilant to auto play attr on player_loc tag.
|
||||
* @example 'ap=1'
|
||||
*/
|
||||
'player_loc:autoplay'?: string;
|
||||
/**
|
||||
* Whether the search engine can embed the video in search results. Allowed values are yes or no.
|
||||
*/
|
||||
'player_loc:allow_embed'?: EnumYesNo;
|
||||
/**
|
||||
* The length of the video in seconds
|
||||
* @example 600
|
||||
*/
|
||||
duration?: number;
|
||||
/**
|
||||
* The date after which the video will no longer be available.
|
||||
* @example "2012-07-16T19:20:30+08:00"
|
||||
*/
|
||||
expiration_date?: string;
|
||||
/**
|
||||
* The number of times the video has been viewed
|
||||
*/
|
||||
view_count?: number;
|
||||
/**
|
||||
* The date the video was first published, in W3C format.
|
||||
* @example "2012-07-16T19:20:30+08:00"
|
||||
*/
|
||||
publication_date?: string;
|
||||
/**
|
||||
* A short description of the broad category that the video belongs to. This is a string no longer than 256 characters.
|
||||
* @example Baking
|
||||
*/
|
||||
category?: string;
|
||||
/**
|
||||
* Whether to show or hide your video in search results from specific countries.
|
||||
* @example "IE GB US CA"
|
||||
*/
|
||||
restriction?: string;
|
||||
/**
|
||||
* Whether the countries in restriction are allowed or denied
|
||||
* @example 'deny'
|
||||
*/
|
||||
'restriction:relationship'?: EnumAllowDeny;
|
||||
gallery_loc?: string;
|
||||
/**
|
||||
* [Optional] Specifies the URL of a webpage with additional information about this uploader. This URL must be in the same domain as the <loc> tag.
|
||||
* @see https://developers.google.com/search/docs/advanced/sitemaps/video-sitemaps
|
||||
* @example http://www.example.com/users/grillymcgrillerson
|
||||
*/
|
||||
'uploader:info'?: string;
|
||||
'gallery_loc:title'?: string;
|
||||
/**
|
||||
* The price to download or view the video. Omit this tag for free videos.
|
||||
* @example "1.99"
|
||||
*/
|
||||
price?: string;
|
||||
/**
|
||||
* Specifies the resolution of the purchased version. Supported values are hd and sd.
|
||||
* @example "HD"
|
||||
*/
|
||||
'price:resolution'?: Resolution;
|
||||
/**
|
||||
* Specifies the currency in ISO4217 format.
|
||||
* @example "USD"
|
||||
*/
|
||||
'price:currency'?: string;
|
||||
/**
|
||||
* Specifies the purchase option. Supported values are rend and own.
|
||||
* @example "rent"
|
||||
*/
|
||||
'price:type'?: PriceType;
|
||||
/**
|
||||
* The video uploader's name. Only one <video:uploader> is allowed per video. String value, max 255 characters.
|
||||
* @example "GrillyMcGrillerson"
|
||||
*/
|
||||
uploader?: string;
|
||||
/**
|
||||
* Whether to show or hide your video in search results on specified platform types. This is a list of space-delimited
|
||||
* platform types. See <https://support.google.com/webmasters/answer/80471?hl=en&ref_topic=4581190> for more detail
|
||||
* @example "tv"
|
||||
*/
|
||||
platform?: string;
|
||||
id?: string;
|
||||
'platform:relationship'?: EnumAllowDeny;
|
||||
}
|
||||
export declare type PriceType = 'rent' | 'purchase' | 'RENT' | 'PURCHASE';
|
||||
export declare type Resolution = 'HD' | 'hd' | 'sd' | 'SD';
|
||||
/**
|
||||
* Sitemap video. <https://support.google.com/webmasters/answer/80471?hl=en&ref_topic=4581190>
|
||||
*/
|
||||
export interface VideoItem extends VideoItemBase {
|
||||
/**
|
||||
* An arbitrary string tag describing the video. Tags are generally very short descriptions of key concepts associated
|
||||
* with a video or piece of content.
|
||||
* @example ['Baking']
|
||||
*/
|
||||
tag: string[];
|
||||
/**
|
||||
* The rating of the video. Supported values are float numbers.
|
||||
* @example 2.5
|
||||
*/
|
||||
rating?: number;
|
||||
family_friendly?: EnumYesNo;
|
||||
/**
|
||||
* Indicates whether a subscription (either paid or free) is required to view
|
||||
* the video. Allowed values are yes or no.
|
||||
*/
|
||||
requires_subscription?: EnumYesNo;
|
||||
/**
|
||||
* Indicates whether the video is a live stream. Supported values are yes or no.
|
||||
*/
|
||||
live?: EnumYesNo;
|
||||
}
|
||||
/**
|
||||
* Sitemap video. <https://support.google.com/webmasters/answer/80471?hl=en&ref_topic=4581190>
|
||||
*/
|
||||
export interface VideoItemLoose extends VideoItemBase {
|
||||
/**
|
||||
* An arbitrary string tag describing the video. Tags are generally very short descriptions of key concepts associated
|
||||
* with a video or piece of content.
|
||||
* @example ['Baking']
|
||||
*/
|
||||
tag?: string | string[];
|
||||
/**
|
||||
* The rating of the video. Supported values are float numbers.
|
||||
* @example 2.5
|
||||
*/
|
||||
rating?: string | number;
|
||||
family_friendly?: EnumYesNo | boolean;
|
||||
requires_subscription?: EnumYesNo | boolean;
|
||||
/**
|
||||
* Indicates whether the video is a live stream. Supported values are yes or no.
|
||||
*/
|
||||
live?: EnumYesNo | boolean;
|
||||
}
|
||||
/**
|
||||
* https://support.google.com/webmasters/answer/189077
|
||||
*/
|
||||
export interface LinkItem {
|
||||
/**
|
||||
* @example 'en'
|
||||
*/
|
||||
lang: string;
|
||||
/**
|
||||
* @example 'en-us'
|
||||
*/
|
||||
hreflang?: string;
|
||||
url: string;
|
||||
}
|
||||
export interface IndexItem {
|
||||
url: string;
|
||||
lastmod?: string;
|
||||
}
|
||||
interface SitemapItemBase {
|
||||
lastmod?: string;
|
||||
changefreq?: EnumChangefreq;
|
||||
fullPrecisionPriority?: boolean;
|
||||
priority?: number;
|
||||
news?: NewsItem;
|
||||
expires?: string;
|
||||
androidLink?: string;
|
||||
ampLink?: string;
|
||||
url: string;
|
||||
}
|
||||
/**
|
||||
* Strict options for individual sitemap entries
|
||||
*/
|
||||
export interface SitemapItem extends SitemapItemBase {
|
||||
img: Img[];
|
||||
video: VideoItem[];
|
||||
links: LinkItem[];
|
||||
}
|
||||
/**
|
||||
* Options for individual sitemap entries prior to normalization
|
||||
*/
|
||||
export interface SitemapItemLoose extends SitemapItemBase {
|
||||
video?: VideoItemLoose | VideoItemLoose[];
|
||||
img?: string | Img | (string | Img)[];
|
||||
links?: LinkItem[];
|
||||
lastmodfile?: string | Buffer | URL;
|
||||
lastmodISO?: string;
|
||||
lastmodrealtime?: boolean;
|
||||
}
|
||||
/**
|
||||
* How to handle errors in passed in urls
|
||||
*/
|
||||
export declare enum ErrorLevel {
|
||||
/**
|
||||
* Validation will be skipped and nothing logged or thrown.
|
||||
*/
|
||||
SILENT = "silent",
|
||||
/**
|
||||
* If an invalid value is encountered, a console.warn will be called with details
|
||||
*/
|
||||
WARN = "warn",
|
||||
/**
|
||||
* An Error will be thrown on encountering invalid data.
|
||||
*/
|
||||
THROW = "throw"
|
||||
}
|
||||
export declare type ErrorHandler = (error: Error, level: ErrorLevel) => void;
|
||||
export declare enum TagNames {
|
||||
url = "url",
|
||||
loc = "loc",
|
||||
urlset = "urlset",
|
||||
lastmod = "lastmod",
|
||||
changefreq = "changefreq",
|
||||
priority = "priority",
|
||||
'video:thumbnail_loc' = "video:thumbnail_loc",
|
||||
'video:video' = "video:video",
|
||||
'video:title' = "video:title",
|
||||
'video:description' = "video:description",
|
||||
'video:tag' = "video:tag",
|
||||
'video:duration' = "video:duration",
|
||||
'video:player_loc' = "video:player_loc",
|
||||
'video:content_loc' = "video:content_loc",
|
||||
'image:image' = "image:image",
|
||||
'image:loc' = "image:loc",
|
||||
'image:geo_location' = "image:geo_location",
|
||||
'image:license' = "image:license",
|
||||
'image:title' = "image:title",
|
||||
'image:caption' = "image:caption",
|
||||
'video:requires_subscription' = "video:requires_subscription",
|
||||
'video:publication_date' = "video:publication_date",
|
||||
'video:id' = "video:id",
|
||||
'video:restriction' = "video:restriction",
|
||||
'video:family_friendly' = "video:family_friendly",
|
||||
'video:view_count' = "video:view_count",
|
||||
'video:uploader' = "video:uploader",
|
||||
'video:expiration_date' = "video:expiration_date",
|
||||
'video:platform' = "video:platform",
|
||||
'video:price' = "video:price",
|
||||
'video:rating' = "video:rating",
|
||||
'video:category' = "video:category",
|
||||
'video:live' = "video:live",
|
||||
'video:gallery_loc' = "video:gallery_loc",
|
||||
'news:news' = "news:news",
|
||||
'news:publication' = "news:publication",
|
||||
'news:name' = "news:name",
|
||||
'news:access' = "news:access",
|
||||
'news:genres' = "news:genres",
|
||||
'news:publication_date' = "news:publication_date",
|
||||
'news:title' = "news:title",
|
||||
'news:keywords' = "news:keywords",
|
||||
'news:stock_tickers' = "news:stock_tickers",
|
||||
'news:language' = "news:language",
|
||||
'mobile:mobile' = "mobile:mobile",
|
||||
'xhtml:link' = "xhtml:link",
|
||||
'expires' = "expires"
|
||||
}
|
||||
export declare enum IndexTagNames {
|
||||
sitemap = "sitemap",
|
||||
sitemapindex = "sitemapindex",
|
||||
loc = "loc",
|
||||
lastmod = "lastmod"
|
||||
}
|
||||
export {};
|
||||
143
node_modules/sitemap/dist/lib/types.js
generated
vendored
Normal file
143
node_modules/sitemap/dist/lib/types.js
generated
vendored
Normal file
@@ -0,0 +1,143 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.IndexTagNames = exports.TagNames = exports.ErrorLevel = exports.isAllowDeny = exports.EnumAllowDeny = exports.isValidYesNo = exports.EnumYesNo = exports.isValidChangeFreq = exports.CHANGEFREQ = exports.isResolution = exports.isPriceType = exports.validators = exports.EnumChangefreq = void 0;
|
||||
/**
|
||||
* How frequently the page is likely to change. This value provides general
|
||||
* information to search engines and may not correlate exactly to how often they crawl the page. Please note that the
|
||||
* value of this tag is considered a hint and not a command. See
|
||||
* <https://www.sitemaps.org/protocol.html#xmlTagDefinitions> for the acceptable
|
||||
* values
|
||||
*/
|
||||
var EnumChangefreq;
|
||||
(function (EnumChangefreq) {
|
||||
EnumChangefreq["DAILY"] = "daily";
|
||||
EnumChangefreq["MONTHLY"] = "monthly";
|
||||
EnumChangefreq["ALWAYS"] = "always";
|
||||
EnumChangefreq["HOURLY"] = "hourly";
|
||||
EnumChangefreq["WEEKLY"] = "weekly";
|
||||
EnumChangefreq["YEARLY"] = "yearly";
|
||||
EnumChangefreq["NEVER"] = "never";
|
||||
})(EnumChangefreq = exports.EnumChangefreq || (exports.EnumChangefreq = {}));
|
||||
const allowDeny = /^(?:allow|deny)$/;
|
||||
exports.validators = {
|
||||
'price:currency': /^[A-Z]{3}$/,
|
||||
'price:type': /^(?:rent|purchase|RENT|PURCHASE)$/,
|
||||
'price:resolution': /^(?:HD|hd|sd|SD)$/,
|
||||
'platform:relationship': allowDeny,
|
||||
'restriction:relationship': allowDeny,
|
||||
restriction: /^([A-Z]{2}( +[A-Z]{2})*)?$/,
|
||||
platform: /^((web|mobile|tv)( (web|mobile|tv))*)?$/,
|
||||
language: /^zh-cn|zh-tw|([a-z]{2,3})$/,
|
||||
genres: /^(PressRelease|Satire|Blog|OpEd|Opinion|UserGenerated)(, *(PressRelease|Satire|Blog|OpEd|Opinion|UserGenerated))*$/,
|
||||
stock_tickers: /^(\w+:\w+(, *\w+:\w+){0,4})?$/,
|
||||
};
|
||||
function isPriceType(pt) {
|
||||
return exports.validators['price:type'].test(pt);
|
||||
}
|
||||
exports.isPriceType = isPriceType;
|
||||
function isResolution(res) {
|
||||
return exports.validators['price:resolution'].test(res);
|
||||
}
|
||||
exports.isResolution = isResolution;
|
||||
exports.CHANGEFREQ = Object.values(EnumChangefreq);
|
||||
function isValidChangeFreq(freq) {
|
||||
return exports.CHANGEFREQ.includes(freq);
|
||||
}
|
||||
exports.isValidChangeFreq = isValidChangeFreq;
|
||||
var EnumYesNo;
|
||||
(function (EnumYesNo) {
|
||||
EnumYesNo["YES"] = "YES";
|
||||
EnumYesNo["NO"] = "NO";
|
||||
EnumYesNo["Yes"] = "Yes";
|
||||
EnumYesNo["No"] = "No";
|
||||
EnumYesNo["yes"] = "yes";
|
||||
EnumYesNo["no"] = "no";
|
||||
})(EnumYesNo = exports.EnumYesNo || (exports.EnumYesNo = {}));
|
||||
function isValidYesNo(yn) {
|
||||
return /^YES|NO|[Yy]es|[Nn]o$/.test(yn);
|
||||
}
|
||||
exports.isValidYesNo = isValidYesNo;
|
||||
var EnumAllowDeny;
|
||||
(function (EnumAllowDeny) {
|
||||
EnumAllowDeny["ALLOW"] = "allow";
|
||||
EnumAllowDeny["DENY"] = "deny";
|
||||
})(EnumAllowDeny = exports.EnumAllowDeny || (exports.EnumAllowDeny = {}));
|
||||
function isAllowDeny(ad) {
|
||||
return allowDeny.test(ad);
|
||||
}
|
||||
exports.isAllowDeny = isAllowDeny;
|
||||
/**
|
||||
* How to handle errors in passed in urls
|
||||
*/
|
||||
var ErrorLevel;
|
||||
(function (ErrorLevel) {
|
||||
/**
|
||||
* Validation will be skipped and nothing logged or thrown.
|
||||
*/
|
||||
ErrorLevel["SILENT"] = "silent";
|
||||
/**
|
||||
* If an invalid value is encountered, a console.warn will be called with details
|
||||
*/
|
||||
ErrorLevel["WARN"] = "warn";
|
||||
/**
|
||||
* An Error will be thrown on encountering invalid data.
|
||||
*/
|
||||
ErrorLevel["THROW"] = "throw";
|
||||
})(ErrorLevel = exports.ErrorLevel || (exports.ErrorLevel = {}));
|
||||
var TagNames;
|
||||
(function (TagNames) {
|
||||
TagNames["url"] = "url";
|
||||
TagNames["loc"] = "loc";
|
||||
TagNames["urlset"] = "urlset";
|
||||
TagNames["lastmod"] = "lastmod";
|
||||
TagNames["changefreq"] = "changefreq";
|
||||
TagNames["priority"] = "priority";
|
||||
TagNames["video:thumbnail_loc"] = "video:thumbnail_loc";
|
||||
TagNames["video:video"] = "video:video";
|
||||
TagNames["video:title"] = "video:title";
|
||||
TagNames["video:description"] = "video:description";
|
||||
TagNames["video:tag"] = "video:tag";
|
||||
TagNames["video:duration"] = "video:duration";
|
||||
TagNames["video:player_loc"] = "video:player_loc";
|
||||
TagNames["video:content_loc"] = "video:content_loc";
|
||||
TagNames["image:image"] = "image:image";
|
||||
TagNames["image:loc"] = "image:loc";
|
||||
TagNames["image:geo_location"] = "image:geo_location";
|
||||
TagNames["image:license"] = "image:license";
|
||||
TagNames["image:title"] = "image:title";
|
||||
TagNames["image:caption"] = "image:caption";
|
||||
TagNames["video:requires_subscription"] = "video:requires_subscription";
|
||||
TagNames["video:publication_date"] = "video:publication_date";
|
||||
TagNames["video:id"] = "video:id";
|
||||
TagNames["video:restriction"] = "video:restriction";
|
||||
TagNames["video:family_friendly"] = "video:family_friendly";
|
||||
TagNames["video:view_count"] = "video:view_count";
|
||||
TagNames["video:uploader"] = "video:uploader";
|
||||
TagNames["video:expiration_date"] = "video:expiration_date";
|
||||
TagNames["video:platform"] = "video:platform";
|
||||
TagNames["video:price"] = "video:price";
|
||||
TagNames["video:rating"] = "video:rating";
|
||||
TagNames["video:category"] = "video:category";
|
||||
TagNames["video:live"] = "video:live";
|
||||
TagNames["video:gallery_loc"] = "video:gallery_loc";
|
||||
TagNames["news:news"] = "news:news";
|
||||
TagNames["news:publication"] = "news:publication";
|
||||
TagNames["news:name"] = "news:name";
|
||||
TagNames["news:access"] = "news:access";
|
||||
TagNames["news:genres"] = "news:genres";
|
||||
TagNames["news:publication_date"] = "news:publication_date";
|
||||
TagNames["news:title"] = "news:title";
|
||||
TagNames["news:keywords"] = "news:keywords";
|
||||
TagNames["news:stock_tickers"] = "news:stock_tickers";
|
||||
TagNames["news:language"] = "news:language";
|
||||
TagNames["mobile:mobile"] = "mobile:mobile";
|
||||
TagNames["xhtml:link"] = "xhtml:link";
|
||||
TagNames["expires"] = "expires";
|
||||
})(TagNames = exports.TagNames || (exports.TagNames = {}));
|
||||
var IndexTagNames;
|
||||
(function (IndexTagNames) {
|
||||
IndexTagNames["sitemap"] = "sitemap";
|
||||
IndexTagNames["sitemapindex"] = "sitemapindex";
|
||||
IndexTagNames["loc"] = "loc";
|
||||
IndexTagNames["lastmod"] = "lastmod";
|
||||
})(IndexTagNames = exports.IndexTagNames || (exports.IndexTagNames = {}));
|
||||
55
node_modules/sitemap/dist/lib/utils.d.ts
generated
vendored
Normal file
55
node_modules/sitemap/dist/lib/utils.d.ts
generated
vendored
Normal file
@@ -0,0 +1,55 @@
|
||||
/// <reference types="node" />
|
||||
import { Readable, ReadableOptions, TransformOptions } from 'stream';
|
||||
import { SitemapItem, ErrorLevel, SitemapItemLoose, ErrorHandler } from './types';
|
||||
/**
|
||||
* Verifies all data passed in will comply with sitemap spec.
|
||||
* @param conf Options to validate
|
||||
* @param level logging level
|
||||
* @param errorHandler error handling func
|
||||
*/
|
||||
export declare function validateSMIOptions(conf: SitemapItem, level?: ErrorLevel, errorHandler?: ErrorHandler): SitemapItem;
|
||||
/**
|
||||
* Combines multiple streams into one
|
||||
* @param streams the streams to combine
|
||||
*/
|
||||
export declare function mergeStreams(streams: Readable[], options?: TransformOptions): Readable;
|
||||
export interface ReadlineStreamOptions extends ReadableOptions {
|
||||
input: Readable;
|
||||
}
|
||||
/**
|
||||
* Wraps node's ReadLine in a stream
|
||||
*/
|
||||
export declare class ReadlineStream extends Readable {
|
||||
private _source;
|
||||
constructor(options: ReadlineStreamOptions);
|
||||
_read(size: number): void;
|
||||
}
|
||||
/**
|
||||
* Takes a stream likely from fs.createReadStream('./path') and returns a stream
|
||||
* of sitemap items
|
||||
* @param stream a stream of line separated urls.
|
||||
* @param opts.isJSON is the stream line separated JSON. leave undefined to guess
|
||||
*/
|
||||
export declare function lineSeparatedURLsToSitemapOptions(stream: Readable, { isJSON }?: {
|
||||
isJSON?: boolean;
|
||||
}): Readable;
|
||||
/**
|
||||
* Based on lodash's implementation of chunk.
|
||||
*
|
||||
* Copyright JS Foundation and other contributors <https://js.foundation/>
|
||||
*
|
||||
* Based on Underscore.js, copyright Jeremy Ashkenas,
|
||||
* DocumentCloud and Investigative Reporters & Editors <http://underscorejs.org/>
|
||||
*
|
||||
* This software consists of voluntary contributions made by many
|
||||
* individuals. For exact contribution history, see the revision history
|
||||
* available at https://github.com/lodash/lodash
|
||||
*/
|
||||
export declare function chunk(array: any[], size?: number): any[];
|
||||
/**
|
||||
* Converts the passed in sitemap entry into one capable of being consumed by SitemapItem
|
||||
* @param {string | SitemapItemLoose} elem the string or object to be converted
|
||||
* @param {string} hostname
|
||||
* @returns SitemapItemOptions a strict sitemap item option
|
||||
*/
|
||||
export declare function normalizeURL(elem: string | SitemapItemLoose, hostname?: string, lastmodDateOnly?: boolean): SitemapItem;
|
||||
353
node_modules/sitemap/dist/lib/utils.js
generated
vendored
Normal file
353
node_modules/sitemap/dist/lib/utils.js
generated
vendored
Normal file
@@ -0,0 +1,353 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.normalizeURL = exports.chunk = exports.lineSeparatedURLsToSitemapOptions = exports.ReadlineStream = exports.mergeStreams = exports.validateSMIOptions = void 0;
|
||||
/*!
|
||||
* Sitemap
|
||||
* Copyright(c) 2011 Eugene Kalinin
|
||||
* MIT Licensed
|
||||
*/
|
||||
const fs_1 = require("fs");
|
||||
const stream_1 = require("stream");
|
||||
const readline_1 = require("readline");
|
||||
const url_1 = require("url");
|
||||
const types_1 = require("./types");
|
||||
const errors_1 = require("./errors");
|
||||
const types_2 = require("./types");
|
||||
function validate(subject, name, url, level) {
|
||||
Object.keys(subject).forEach((key) => {
|
||||
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
||||
// @ts-ignore
|
||||
const val = subject[key];
|
||||
if (types_2.validators[key] && !types_2.validators[key].test(val)) {
|
||||
if (level === types_1.ErrorLevel.THROW) {
|
||||
throw new errors_1.InvalidAttrValue(key, val, types_2.validators[key]);
|
||||
}
|
||||
else {
|
||||
console.warn(`${url}: ${name} key ${key} has invalid value: ${val}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
function handleError(error, level) {
|
||||
if (level === types_1.ErrorLevel.THROW) {
|
||||
throw error;
|
||||
}
|
||||
else if (level === types_1.ErrorLevel.WARN) {
|
||||
console.warn(error.name, error.message);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Verifies all data passed in will comply with sitemap spec.
|
||||
* @param conf Options to validate
|
||||
* @param level logging level
|
||||
* @param errorHandler error handling func
|
||||
*/
|
||||
function validateSMIOptions(conf, level = types_1.ErrorLevel.WARN, errorHandler = handleError) {
|
||||
if (!conf) {
|
||||
throw new errors_1.NoConfigError();
|
||||
}
|
||||
if (level === types_1.ErrorLevel.SILENT) {
|
||||
return conf;
|
||||
}
|
||||
const { url, changefreq, priority, news, video } = conf;
|
||||
if (!url) {
|
||||
errorHandler(new errors_1.NoURLError(), level);
|
||||
}
|
||||
if (changefreq) {
|
||||
if (!(0, types_1.isValidChangeFreq)(changefreq)) {
|
||||
errorHandler(new errors_1.ChangeFreqInvalidError(url, changefreq), level);
|
||||
}
|
||||
}
|
||||
if (priority) {
|
||||
if (!(priority >= 0.0 && priority <= 1.0)) {
|
||||
errorHandler(new errors_1.PriorityInvalidError(url, priority), level);
|
||||
}
|
||||
}
|
||||
if (news) {
|
||||
if (news.access &&
|
||||
news.access !== 'Registration' &&
|
||||
news.access !== 'Subscription') {
|
||||
errorHandler(new errors_1.InvalidNewsAccessValue(url, news.access), level);
|
||||
}
|
||||
if (!news.publication ||
|
||||
!news.publication.name ||
|
||||
!news.publication.language ||
|
||||
!news.publication_date ||
|
||||
!news.title) {
|
||||
errorHandler(new errors_1.InvalidNewsFormat(url), level);
|
||||
}
|
||||
validate(news, 'news', url, level);
|
||||
validate(news.publication, 'publication', url, level);
|
||||
}
|
||||
if (video) {
|
||||
video.forEach((vid) => {
|
||||
var _a;
|
||||
if (vid.duration !== undefined) {
|
||||
if (vid.duration < 0 || vid.duration > 28800) {
|
||||
errorHandler(new errors_1.InvalidVideoDuration(url, vid.duration), level);
|
||||
}
|
||||
}
|
||||
if (vid.rating !== undefined && (vid.rating < 0 || vid.rating > 5)) {
|
||||
errorHandler(new errors_1.InvalidVideoRating(url, vid.title, vid.rating), level);
|
||||
}
|
||||
if (typeof vid !== 'object' ||
|
||||
!vid.thumbnail_loc ||
|
||||
!vid.title ||
|
||||
!vid.description) {
|
||||
// has to be an object and include required categories https://support.google.com/webmasters/answer/80471?hl=en&ref_topic=4581190
|
||||
errorHandler(new errors_1.InvalidVideoFormat(url), level);
|
||||
}
|
||||
if (vid.title.length > 100) {
|
||||
errorHandler(new errors_1.InvalidVideoTitle(url, vid.title.length), level);
|
||||
}
|
||||
if (vid.description.length > 2048) {
|
||||
errorHandler(new errors_1.InvalidVideoDescription(url, vid.description.length), level);
|
||||
}
|
||||
if (vid.view_count !== undefined && vid.view_count < 0) {
|
||||
errorHandler(new errors_1.InvalidVideoViewCount(url, vid.view_count), level);
|
||||
}
|
||||
if (vid.tag.length > 32) {
|
||||
errorHandler(new errors_1.InvalidVideoTagCount(url, vid.tag.length), level);
|
||||
}
|
||||
if (vid.category !== undefined && ((_a = vid.category) === null || _a === void 0 ? void 0 : _a.length) > 256) {
|
||||
errorHandler(new errors_1.InvalidVideoCategory(url, vid.category.length), level);
|
||||
}
|
||||
if (vid.family_friendly !== undefined &&
|
||||
!(0, types_1.isValidYesNo)(vid.family_friendly)) {
|
||||
errorHandler(new errors_1.InvalidVideoFamilyFriendly(url, vid.family_friendly), level);
|
||||
}
|
||||
if (vid.restriction) {
|
||||
if (!types_2.validators.restriction.test(vid.restriction)) {
|
||||
errorHandler(new errors_1.InvalidVideoRestriction(url, vid.restriction), level);
|
||||
}
|
||||
if (!vid['restriction:relationship'] ||
|
||||
!(0, types_1.isAllowDeny)(vid['restriction:relationship'])) {
|
||||
errorHandler(new errors_1.InvalidVideoRestrictionRelationship(url, vid['restriction:relationship']), level);
|
||||
}
|
||||
}
|
||||
// TODO price element should be unbounded
|
||||
if ((vid.price === '' && vid['price:type'] === undefined) ||
|
||||
(vid['price:type'] !== undefined && !(0, types_1.isPriceType)(vid['price:type']))) {
|
||||
errorHandler(new errors_1.InvalidVideoPriceType(url, vid['price:type'], vid.price), level);
|
||||
}
|
||||
if (vid['price:resolution'] !== undefined &&
|
||||
!(0, types_1.isResolution)(vid['price:resolution'])) {
|
||||
errorHandler(new errors_1.InvalidVideoResolution(url, vid['price:resolution']), level);
|
||||
}
|
||||
if (vid['price:currency'] !== undefined &&
|
||||
!types_2.validators['price:currency'].test(vid['price:currency'])) {
|
||||
errorHandler(new errors_1.InvalidVideoPriceCurrency(url, vid['price:currency']), level);
|
||||
}
|
||||
validate(vid, 'video', url, level);
|
||||
});
|
||||
}
|
||||
return conf;
|
||||
}
|
||||
exports.validateSMIOptions = validateSMIOptions;
|
||||
/**
|
||||
* Combines multiple streams into one
|
||||
* @param streams the streams to combine
|
||||
*/
|
||||
function mergeStreams(streams, options) {
|
||||
let pass = new stream_1.PassThrough(options);
|
||||
let waiting = streams.length;
|
||||
for (const stream of streams) {
|
||||
pass = stream.pipe(pass, { end: false });
|
||||
stream.once('end', () => --waiting === 0 && pass.emit('end'));
|
||||
}
|
||||
return pass;
|
||||
}
|
||||
exports.mergeStreams = mergeStreams;
|
||||
/**
|
||||
* Wraps node's ReadLine in a stream
|
||||
*/
|
||||
class ReadlineStream extends stream_1.Readable {
|
||||
constructor(options) {
|
||||
if (options.autoDestroy === undefined) {
|
||||
options.autoDestroy = true;
|
||||
}
|
||||
options.objectMode = true;
|
||||
super(options);
|
||||
this._source = (0, readline_1.createInterface)({
|
||||
input: options.input,
|
||||
terminal: false,
|
||||
crlfDelay: Infinity,
|
||||
});
|
||||
// Every time there's data, push it into the internal buffer.
|
||||
this._source.on('line', (chunk) => {
|
||||
// If push() returns false, then stop reading from source.
|
||||
if (!this.push(chunk))
|
||||
this._source.pause();
|
||||
});
|
||||
// When the source ends, push the EOF-signaling `null` chunk.
|
||||
this._source.on('close', () => {
|
||||
this.push(null);
|
||||
});
|
||||
}
|
||||
// _read() will be called when the stream wants to pull more data in.
|
||||
// The advisory size argument is ignored in this case.
|
||||
_read(size) {
|
||||
this._source.resume();
|
||||
}
|
||||
}
|
||||
exports.ReadlineStream = ReadlineStream;
|
||||
/**
|
||||
* Takes a stream likely from fs.createReadStream('./path') and returns a stream
|
||||
* of sitemap items
|
||||
* @param stream a stream of line separated urls.
|
||||
* @param opts.isJSON is the stream line separated JSON. leave undefined to guess
|
||||
*/
|
||||
function lineSeparatedURLsToSitemapOptions(stream, { isJSON } = {}) {
|
||||
return new ReadlineStream({ input: stream }).pipe(new stream_1.Transform({
|
||||
objectMode: true,
|
||||
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
|
||||
transform: (line, encoding, cb) => {
|
||||
if (isJSON || (isJSON === undefined && line[0] === '{')) {
|
||||
cb(null, JSON.parse(line));
|
||||
}
|
||||
else {
|
||||
cb(null, line);
|
||||
}
|
||||
},
|
||||
}));
|
||||
}
|
||||
exports.lineSeparatedURLsToSitemapOptions = lineSeparatedURLsToSitemapOptions;
|
||||
/**
|
||||
* Based on lodash's implementation of chunk.
|
||||
*
|
||||
* Copyright JS Foundation and other contributors <https://js.foundation/>
|
||||
*
|
||||
* Based on Underscore.js, copyright Jeremy Ashkenas,
|
||||
* DocumentCloud and Investigative Reporters & Editors <http://underscorejs.org/>
|
||||
*
|
||||
* This software consists of voluntary contributions made by many
|
||||
* individuals. For exact contribution history, see the revision history
|
||||
* available at https://github.com/lodash/lodash
|
||||
*/
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||
function chunk(array, size = 1) {
|
||||
size = Math.max(Math.trunc(size), 0);
|
||||
const length = array ? array.length : 0;
|
||||
if (!length || size < 1) {
|
||||
return [];
|
||||
}
|
||||
const result = Array(Math.ceil(length / size));
|
||||
let index = 0, resIndex = 0;
|
||||
while (index < length) {
|
||||
result[resIndex++] = array.slice(index, (index += size));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
exports.chunk = chunk;
|
||||
function boolToYESNO(bool) {
|
||||
if (bool === undefined) {
|
||||
return bool;
|
||||
}
|
||||
if (typeof bool === 'boolean') {
|
||||
return bool ? types_1.EnumYesNo.yes : types_1.EnumYesNo.no;
|
||||
}
|
||||
return bool;
|
||||
}
|
||||
/**
|
||||
* Converts the passed in sitemap entry into one capable of being consumed by SitemapItem
|
||||
* @param {string | SitemapItemLoose} elem the string or object to be converted
|
||||
* @param {string} hostname
|
||||
* @returns SitemapItemOptions a strict sitemap item option
|
||||
*/
|
||||
function normalizeURL(elem, hostname, lastmodDateOnly = false) {
|
||||
// SitemapItem
|
||||
// create object with url property
|
||||
let smi = {
|
||||
img: [],
|
||||
video: [],
|
||||
links: [],
|
||||
url: '',
|
||||
};
|
||||
let smiLoose;
|
||||
if (typeof elem === 'string') {
|
||||
smi.url = elem;
|
||||
smiLoose = { url: elem };
|
||||
}
|
||||
else {
|
||||
smiLoose = elem;
|
||||
}
|
||||
smi.url = new url_1.URL(smiLoose.url, hostname).toString();
|
||||
let img = [];
|
||||
if (smiLoose.img) {
|
||||
if (typeof smiLoose.img === 'string') {
|
||||
// string -> array of objects
|
||||
smiLoose.img = [{ url: smiLoose.img }];
|
||||
}
|
||||
else if (!Array.isArray(smiLoose.img)) {
|
||||
// object -> array of objects
|
||||
smiLoose.img = [smiLoose.img];
|
||||
}
|
||||
img = smiLoose.img.map((el) => (typeof el === 'string' ? { url: el } : el));
|
||||
}
|
||||
// prepend hostname to all image urls
|
||||
smi.img = img.map((el) => ({
|
||||
...el,
|
||||
url: new url_1.URL(el.url, hostname).toString(),
|
||||
}));
|
||||
let links = [];
|
||||
if (smiLoose.links) {
|
||||
links = smiLoose.links;
|
||||
}
|
||||
smi.links = links.map((link) => {
|
||||
return { ...link, url: new url_1.URL(link.url, hostname).toString() };
|
||||
});
|
||||
if (smiLoose.video) {
|
||||
if (!Array.isArray(smiLoose.video)) {
|
||||
// make it an array
|
||||
smiLoose.video = [smiLoose.video];
|
||||
}
|
||||
smi.video = smiLoose.video.map((video) => {
|
||||
const nv = {
|
||||
...video,
|
||||
family_friendly: boolToYESNO(video.family_friendly),
|
||||
live: boolToYESNO(video.live),
|
||||
requires_subscription: boolToYESNO(video.requires_subscription),
|
||||
tag: [],
|
||||
rating: undefined,
|
||||
};
|
||||
if (video.tag !== undefined) {
|
||||
nv.tag = !Array.isArray(video.tag) ? [video.tag] : video.tag;
|
||||
}
|
||||
if (video.rating !== undefined) {
|
||||
if (typeof video.rating === 'string') {
|
||||
nv.rating = parseFloat(video.rating);
|
||||
}
|
||||
else {
|
||||
nv.rating = video.rating;
|
||||
}
|
||||
}
|
||||
if (typeof video.view_count === 'string') {
|
||||
nv.view_count = parseInt(video.view_count, 10);
|
||||
}
|
||||
else if (typeof video.view_count === 'number') {
|
||||
nv.view_count = video.view_count;
|
||||
}
|
||||
return nv;
|
||||
});
|
||||
}
|
||||
// If given a file to use for last modified date
|
||||
if (smiLoose.lastmodfile) {
|
||||
const { mtime } = (0, fs_1.statSync)(smiLoose.lastmodfile);
|
||||
smi.lastmod = new Date(mtime).toISOString();
|
||||
// The date of last modification (YYYY-MM-DD)
|
||||
}
|
||||
else if (smiLoose.lastmodISO) {
|
||||
smi.lastmod = new Date(smiLoose.lastmodISO).toISOString();
|
||||
}
|
||||
else if (smiLoose.lastmod) {
|
||||
smi.lastmod = new Date(smiLoose.lastmod).toISOString();
|
||||
}
|
||||
if (lastmodDateOnly && smi.lastmod) {
|
||||
smi.lastmod = smi.lastmod.slice(0, 10);
|
||||
}
|
||||
delete smiLoose.lastmodfile;
|
||||
delete smiLoose.lastmodISO;
|
||||
smi = { ...smiLoose, ...smi };
|
||||
return smi;
|
||||
}
|
||||
exports.normalizeURL = normalizeURL;
|
||||
8
node_modules/sitemap/dist/lib/xmllint.d.ts
generated
vendored
Normal file
8
node_modules/sitemap/dist/lib/xmllint.d.ts
generated
vendored
Normal file
@@ -0,0 +1,8 @@
|
||||
/// <reference types="node" />
|
||||
import { Readable } from 'stream';
|
||||
/**
|
||||
* Verify the passed in xml is valid. Requires xmllib be installed
|
||||
* @param xml what you want validated
|
||||
* @return {Promise<void>} resolves on valid rejects [error stderr]
|
||||
*/
|
||||
export declare function xmlLint(xml: string | Readable): Promise<void>;
|
||||
43
node_modules/sitemap/dist/lib/xmllint.js
generated
vendored
Normal file
43
node_modules/sitemap/dist/lib/xmllint.js
generated
vendored
Normal file
@@ -0,0 +1,43 @@
|
||||
"use strict";
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.xmlLint = void 0;
|
||||
const path_1 = require("path");
|
||||
const child_process_1 = require("child_process");
|
||||
const errors_1 = require("./errors");
|
||||
/**
|
||||
* Verify the passed in xml is valid. Requires xmllib be installed
|
||||
* @param xml what you want validated
|
||||
* @return {Promise<void>} resolves on valid rejects [error stderr]
|
||||
*/
|
||||
function xmlLint(xml) {
|
||||
const args = [
|
||||
'--schema',
|
||||
(0, path_1.resolve)(__dirname, '..', '..', 'schema', 'all.xsd'),
|
||||
'--noout',
|
||||
'-',
|
||||
];
|
||||
if (typeof xml === 'string') {
|
||||
args[args.length - 1] = xml;
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
(0, child_process_1.execFile)('which', ['xmllint'], (error, stdout, stderr) => {
|
||||
if (error) {
|
||||
reject([new errors_1.XMLLintUnavailable()]);
|
||||
return;
|
||||
}
|
||||
const xmllint = (0, child_process_1.execFile)('xmllint', args, (error, stdout, stderr) => {
|
||||
if (error) {
|
||||
reject([error, stderr]);
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
if (xmllint.stdout) {
|
||||
xmllint.stdout.unpipe();
|
||||
if (typeof xml !== 'string' && xml && xmllint.stdin) {
|
||||
xml.pipe(xmllint.stdin);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
exports.xmlLint = xmlLint;
|
||||
Reference in New Issue
Block a user