From 7fa0388f0548d9eccb8352d3d069eaab57d6179e Mon Sep 17 00:00:00 2001 From: "FiveFilters.org" Date: Tue, 17 Aug 2021 01:46:03 +0200 Subject: Update test data --- test/ReadabilityTest.php | 2 +- .../expected-images.json | 12 +- .../base-url-base-element/expected-images.json | 10 +- test/test-pages/base-url/expected-images.json | 12 +- test/test-pages/bbc-1/expected-images.json | 14 +- test/test-pages/iab-1/expected-images.json | 8 +- test/test-pages/keep-images/expected-images.json | 34 +- .../keep-tabular-data/expected-images.json | 19 + .../keep-tabular-data/expected-metadata.json | 8 + test/test-pages/keep-tabular-data/expected.html | 537 +++++++ test/test-pages/keep-tabular-data/source.html | 740 +++++++++ .../expected-images.json | 22 +- .../lifehacker-working/expected-images.json | 22 +- test/test-pages/nytimes-3/expected-images.json | 6 + test/test-pages/nytimes-3/expected-metadata.json | 8 + test/test-pages/nytimes-3/expected.html | 335 ++++ test/test-pages/nytimes-3/source.html | 1638 ++++++++++++++++++++ 17 files changed, 3359 insertions(+), 68 deletions(-) create mode 100644 test/test-pages/keep-tabular-data/expected-images.json create mode 100644 test/test-pages/keep-tabular-data/expected-metadata.json create mode 100644 test/test-pages/keep-tabular-data/expected.html create mode 100644 test/test-pages/keep-tabular-data/source.html create mode 100644 test/test-pages/nytimes-3/expected-images.json create mode 100644 test/test-pages/nytimes-3/expected-metadata.json create mode 100644 test/test-pages/nytimes-3/expected.html create mode 100644 test/test-pages/nytimes-3/source.html diff --git a/test/ReadabilityTest.php b/test/ReadabilityTest.php index 3ef121f..842d259 100644 --- a/test/ReadabilityTest.php +++ b/test/ReadabilityTest.php @@ -95,7 +95,7 @@ class ReadabilityTest extends \PHPUnit\Framework\TestCase $readability = new Readability($configuration); $readability->parse($testPage->getSourceHTML()); - $this->assertSame($testPage->getExpectedImages(), $readability->getImages()); + $this->assertSame($testPage->getExpectedImages(), array_values($readability->getImages())); } /** diff --git a/test/test-pages/base-url-base-element-relative/expected-images.json b/test/test-pages/base-url-base-element-relative/expected-images.json index 5e3fa41..51ed665 100644 --- a/test/test-pages/base-url-base-element-relative/expected-images.json +++ b/test/test-pages/base-url-base-element-relative/expected-images.json @@ -1,6 +1,6 @@ -{ - "0": "http:\/\/fakehost\/test\/base\/foo\/bar\/baz.png", - "2": "http:\/\/fakehost\/foo\/bar\/baz.png", - "3": "http:\/\/test\/foo\/bar\/baz.png", - "4": "https:\/\/test\/foo\/bar\/baz.png" -} \ No newline at end of file +[ + "http:\/\/fakehost\/test\/base\/foo\/bar\/baz.png", + "http:\/\/fakehost\/foo\/bar\/baz.png", + "http:\/\/test\/foo\/bar\/baz.png", + "https:\/\/test\/foo\/bar\/baz.png" +] \ No newline at end of file diff --git a/test/test-pages/base-url-base-element/expected-images.json b/test/test-pages/base-url-base-element/expected-images.json index c4a597e..68c4864 100644 --- a/test/test-pages/base-url-base-element/expected-images.json +++ b/test/test-pages/base-url-base-element/expected-images.json @@ -1,5 +1,5 @@ -{ - "0": "http:\/\/fakehost\/foo\/bar\/baz.png", - "3": "http:\/\/test\/foo\/bar\/baz.png", - "4": "https:\/\/test\/foo\/bar\/baz.png" -} \ No newline at end of file +[ + "http:\/\/fakehost\/foo\/bar\/baz.png", + "http:\/\/test\/foo\/bar\/baz.png", + "https:\/\/test\/foo\/bar\/baz.png" +] \ No newline at end of file diff --git a/test/test-pages/base-url/expected-images.json b/test/test-pages/base-url/expected-images.json index 1c28ae6..e072ac1 100644 --- a/test/test-pages/base-url/expected-images.json +++ b/test/test-pages/base-url/expected-images.json @@ -1,6 +1,6 @@ -{ - "0": "http:\/\/fakehost\/test\/foo\/bar\/baz.png", - "2": "http:\/\/fakehost\/foo\/bar\/baz.png", - "3": "http:\/\/test\/foo\/bar\/baz.png", - "4": "https:\/\/test\/foo\/bar\/baz.png" -} \ No newline at end of file +[ + "http:\/\/fakehost\/test\/foo\/bar\/baz.png", + "http:\/\/fakehost\/foo\/bar\/baz.png", + "http:\/\/test\/foo\/bar\/baz.png", + "https:\/\/test\/foo\/bar\/baz.png" +] \ No newline at end of file diff --git a/test/test-pages/bbc-1/expected-images.json b/test/test-pages/bbc-1/expected-images.json index 44bca8d..5c50d84 100644 --- a/test/test-pages/bbc-1/expected-images.json +++ b/test/test-pages/bbc-1/expected-images.json @@ -1,7 +1,7 @@ -{ - "0": "http:\/\/ichef.bbci.co.uk\/news\/1024\/cpsprodpb\/3D8B\/production\/_84455751_84455749.jpg", - "1": "http:\/\/ichef.bbci.co.uk\/news\/555\/cpsprodpb\/462D\/production\/_84456971_gettyimages-167501087.jpg", - "2": "http:\/\/ichef.bbci.co.uk\/news\/555\/media\/images\/76020000\/jpg\/_76020974_line976.jpg", - "3": "http:\/\/ichef-1.bbci.co.uk\/news\/555\/cpsprodpb\/6D3D\/production\/_84456972_p072315al-0500.jpg", - "5": "http:\/\/ichef-1.bbci.co.uk\/news\/555\/cpsprodpb\/142FD\/production\/_84458628_shirtreuters.jpg" -} \ No newline at end of file +[ + "http:\/\/ichef.bbci.co.uk\/news\/1024\/cpsprodpb\/3D8B\/production\/_84455751_84455749.jpg", + "http:\/\/ichef.bbci.co.uk\/news\/555\/cpsprodpb\/462D\/production\/_84456971_gettyimages-167501087.jpg", + "http:\/\/ichef.bbci.co.uk\/news\/555\/media\/images\/76020000\/jpg\/_76020974_line976.jpg", + "http:\/\/ichef-1.bbci.co.uk\/news\/555\/cpsprodpb\/6D3D\/production\/_84456972_p072315al-0500.jpg", + "http:\/\/ichef-1.bbci.co.uk\/news\/555\/cpsprodpb\/142FD\/production\/_84458628_shirtreuters.jpg" +] \ No newline at end of file diff --git a/test/test-pages/iab-1/expected-images.json b/test/test-pages/iab-1/expected-images.json index f548bd9..105a380 100644 --- a/test/test-pages/iab-1/expected-images.json +++ b/test/test-pages/iab-1/expected-images.json @@ -1,4 +1,4 @@ -{ - "0": "http:\/\/www.iab.com\/wp-content\/uploads\/2015\/10\/getting-lean-with-digital-ad-ux-300x250.jpg", - "2": "http:\/\/www.iab.com\/wp-content\/uploads\/2015\/05\/auto-draft-16-150x150.jpg" -} \ No newline at end of file +[ + "http:\/\/www.iab.com\/wp-content\/uploads\/2015\/10\/getting-lean-with-digital-ad-ux-300x250.jpg", + "http:\/\/www.iab.com\/wp-content\/uploads\/2015\/05\/auto-draft-16-150x150.jpg" +] \ No newline at end of file diff --git a/test/test-pages/keep-images/expected-images.json b/test/test-pages/keep-images/expected-images.json index d5b5aff..e5097f4 100644 --- a/test/test-pages/keep-images/expected-images.json +++ b/test/test-pages/keep-images/expected-images.json @@ -1,17 +1,17 @@ -{ - "0": "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/800\/1*sLDnS1UWEFIS33uLMxq3cw.jpeg", - "1": "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/2000\/1*sLDnS1UWEFIS33uLMxq3cw.jpeg", - "2": "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/800\/1*3vIhkoHIzcxvUdijoCVx6w.png", - "3": "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/2000\/1*4gN1-fzOwCniw-DbqQjDeQ.jpeg", - "4": "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/2000\/1*2KPmZkIBUrhps-2uwDvYFQ.jpeg", - "5": "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/2000\/1*PU40bbbox2Ompc5I3RE99A.jpeg", - "6": "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/800\/1*ohyycinH18fz98TCyUzVgQ.png", - "7": "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/2000\/1*mKvUNOAVQxl6atCbxbCZsg.jpeg", - "8": "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/2000\/1*knT10_FNVUmqQIBLnutmzQ.jpeg", - "10": "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/2000\/1*Vr61dyCTRwk6CemmVF8YAQ.jpeg", - "11": "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/2000\/1*a-1_13xE6_ErQ-QSlz6myw.jpeg", - "13": "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/2000\/1*IWXhtSsVv0gNnCwnDEXk-Q.jpeg", - "14": "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/2000\/1*NGcrjfkV0l37iQH2uyYjEw.jpeg", - "15": "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/2000\/1*WRlKt3q3mt7utmwxcbl3sQ.jpeg", - "16": "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/800\/1*320_4I0lxbn5x3bx4XPI5Q.png" -} \ No newline at end of file +[ + "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/800\/1*sLDnS1UWEFIS33uLMxq3cw.jpeg", + "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/2000\/1*sLDnS1UWEFIS33uLMxq3cw.jpeg", + "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/800\/1*3vIhkoHIzcxvUdijoCVx6w.png", + "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/2000\/1*4gN1-fzOwCniw-DbqQjDeQ.jpeg", + "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/2000\/1*2KPmZkIBUrhps-2uwDvYFQ.jpeg", + "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/2000\/1*PU40bbbox2Ompc5I3RE99A.jpeg", + "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/800\/1*ohyycinH18fz98TCyUzVgQ.png", + "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/2000\/1*mKvUNOAVQxl6atCbxbCZsg.jpeg", + "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/2000\/1*knT10_FNVUmqQIBLnutmzQ.jpeg", + "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/2000\/1*Vr61dyCTRwk6CemmVF8YAQ.jpeg", + "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/2000\/1*a-1_13xE6_ErQ-QSlz6myw.jpeg", + "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/2000\/1*IWXhtSsVv0gNnCwnDEXk-Q.jpeg", + "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/2000\/1*NGcrjfkV0l37iQH2uyYjEw.jpeg", + "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/2000\/1*WRlKt3q3mt7utmwxcbl3sQ.jpeg", + "https:\/\/d262ilb51hltx0.cloudfront.net\/max\/800\/1*320_4I0lxbn5x3bx4XPI5Q.png" +] \ No newline at end of file diff --git a/test/test-pages/keep-tabular-data/expected-images.json b/test/test-pages/keep-tabular-data/expected-images.json new file mode 100644 index 0000000..a748bac --- /dev/null +++ b/test/test-pages/keep-tabular-data/expected-images.json @@ -0,0 +1,19 @@ +[ + "http:\/\/www.factorio.com\/static\/img\/factorio-wheel.png", + "https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-277-finished-2.png", + "https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-newly-finished.png", + "https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-277-not-finished-2.png", + "https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-blueprint-library-grid-view.png", + "https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-blueprint-library-list-view.png", + "https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-blueprint-book.png", + "https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-the-hand.png", + "https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-enemy-bases.gif", + "https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-cliff-controls.gif", + "https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-rectangles.png", + "https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-moisture+aux-debug-map.png", + "https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-moisture+aux-controls.gif", + "https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-island.png", + "https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-islands.png", + "https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-accumulator.gif", + "https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-accumulator-comparison.png" +] \ No newline at end of file diff --git a/test/test-pages/keep-tabular-data/expected-metadata.json b/test/test-pages/keep-tabular-data/expected-metadata.json new file mode 100644 index 0000000..820a3d0 --- /dev/null +++ b/test/test-pages/keep-tabular-data/expected-metadata.json @@ -0,0 +1,8 @@ +{ + "Author": null, + "Direction": null, + "Excerpt": "Posted by kovarex, TOGos, Ernestas, Albert on 2019-02-15, all posts", + "Image": "http:\/\/www.factorio.com\/static\/img\/factorio-wheel.png", + "Title": "Friday Facts #282 - 0.17 in sight | Factorio", + "SiteName": "Factorio.com" +} \ No newline at end of file diff --git a/test/test-pages/keep-tabular-data/expected.html b/test/test-pages/keep-tabular-data/expected.html new file mode 100644 index 0000000..ab10fe6 --- /dev/null +++ b/test/test-pages/keep-tabular-data/expected.html @@ -0,0 +1,537 @@ +
+ +
+
+

+ Friday Facts #282 - 0.17 in sight +

+

Posted by kovarex, TOGos, Ernestas, Albert on 2019-02-15, all posts

+ +

The release plan (kovarex) +

+

This week was the time to close and finish all the things that will go to 0.17.0.

+

Not all of the things that we originally planned to be done were done (surprise), but we just left any non-essential stuff for later so we won't postpone the release any further. The plan is, that next week will be dedicated to the office playtesting and bugfixing. Many would argue, that we could just release instantly and let the players find the bugs for us, but we want to fix the most obvious problems in-house to avoid too many duplicate bug reports and chaos after the release. Also, some potential bugs, like save corruptions, are much more easily worked on in-house.

+

+ If the playtesting goes well, we will let you know next Friday, and if it is the case, we will aim to release the week starting 25th February. +

+

After release plan

+

+ Since there are a lot of things we would like to do before we can call 0.17 good enough, we will simply push new things into the 0.17 releases as time goes on. Even if 0.17 becomes stable in a reasonable time, we would still push things on top of it. We can still make experimental/stable version numbers inside 0.17. Most of the things shouldn't be big enough to make the game generally unstable. I've heard countless times a proposal to make small frequent releases of what have we added, this will probably be reality after 0.17 for some time. +

+

+ The smaller releases will contain mainly: +

+
    +
  • Final looks and behaviour of new GUI screens as they will be finished.
    • +
  • New graphics.
    • +
  • New sounds and sound system tweaks.
    • +
  • Mini tutorial additions and tweaks.
    • +
+ +

+ This is actually quite a large change to our procedures, and there are many ways we will be trying to maximize the effectiveness of smaller and more regular content updates. +

+

The GUI progress (kovarex) +

+

+ There are several GUI screens that are finished. Others (most of them) are just left there as they are in 0.16. They are a combination of the new GUI styles and old ones. They sometimes look funny and out of place, but they should be functional. +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Load map
Save map
Graphics settings
Control settings
Sound settings
Interface settings
Other settings
Map generator
Quick bar Twinsen
Train GUI kovarex
Technology GUI Oxyd
Technology tooltip Oxyd
Blueprint library kovarex
Shortcut bar Oxyd
Character screen Dominik
Help overlay kovarex
Manage/Install mods Rseding
Recipe/item/Entity tooltip Twinsen
Chat icon selector ?
New game ?
Menu structure ?
Main screen chat ?
Recipe explorer ?
+

* Newly finished things since the last update in FFF-277.

Blueprint library

+

+ The blueprint library changes have been split into several steps. The reason is, that there was a big motivation to do the integration with the new quickbar (final version introduced in FFF-278) in time for 0.17.0, while the other changes can be done after. The thing with the quickbar is, that it is quite a big change to one of the most used tools in the game and people generally don't like change even when it is for the better. To minimize the hate of the change, we need to "sell it properly". By that, we should provide as many of the positive aspects of the new quickbar at the time of its introduction. +

+

+ So the change that is already implemented and working for 0.17 is the ability to put blueprints/books into the quick bar in a way that the quick bar is linked directly to the blueprint library window, so you don't need to have the physical blueprint items in your inventory. The other change is, that picking a blueprint from the blueprint library and then pressing Q will just dismiss it, instead of silently pushing it to your inventory. This works the same as the clipboard described in FFF-255. You can still explicitly insert the blueprint from the library to an inventory slot, but if you just pick it, use it, and press Q, it goes away. +

+

+ In addition to this, other changes related to the blueprint library will follow soon after 0.17.0. The first thing is the change of how the GUI looks: +

+

+ +

+

+ We will also allow to switch between grid and list view. It mainly provides a way to nicely see the longer names of the blueprint. We noticed that players try to put a large amount of info about a blueprint in its name, so we are planning to add a possibility to write a textual description of the blueprint. +

+

+ +

+

+ The last big change is to allow to put blueprint books into blueprint books, allowing better organisation. Basically like a directory structure. Whenever a blueprint/book is opened, we plan to show its current location, so the player knows exactly what is going on. +

+

+ +

+

The hand

+

+ Has it ever happened to you, that you have robots trying to put things into your full inventory, while you pick an item from it to build something, and then you just can't put it back, as the diligent robots just filled the last slot in your inventory by whatever they are trying to give to you? Wood from tree removal is the most frequent thing in my case. +

+

+ This was annoying in 0.16 from time to time, but with the new quickbar, it started to happen even more, as now, you have only one inventory, and no reserved slots in the quickbar. To solve that, we just extended the "principal" of the hand. When you pick something from the inventory, the hand icon appears on the slot. As long as you hold the thing in your cursor, the hand stays there, and prevents other things from being inserted there. This way, you should always be able to return the currently selected item into your inventory as long as you didn't get it from external source like a chest. +

+

+
+ The hand is protecting the slot from the robots. +

+

Terrain generation updates (TOGoS) +

+

Everyone has different opinions about what makes a good Factorio world. + I've been working on several changes for 0.17, but the overarching theme + has been to make the map generator options screen more intuitive + and more powerful.

+

+ This was talked about somewhat in an earlier FFF (FFF-258) regarding ore placement, + but since then we found more stuff to fix. +

+

Biter Bases

+

+ In 0.16, the size control for biter bases didn't have much effect. + The frequency control changed the frequency, but that also decreased the size of bases, + which wasn't generally what people wanted. +

+

+ For 0.17 we've reworked biter placement using a system similar to that with which we got resource placement under control. The size and frequency controls now act more like most people would expect, with frequency increasing the number of bases, and size changing the size of each base. +

+

+ +
New preview UI showing the effects of enemy base controls. + In reality the preview takes a couple seconds to regenerate after every change, + but the regeneration part is skipped in this animation to clearly show the effects of the controls. +

+

+ If you don't like the relatively uniform-across-the-world placement of biters, + there are changes under the hood to make it easier for modders to do something different. + Placement is now based on NamedNoiseExpressions "enemy-base-frequency" and "enemy-base-radius", which in turn reference "enemy-base-intensity". + By overriding any of those, a modder could easily create a map where biters are found only at high elevations, + or only near water, or correlate enemy placement with that of resources, or any other thing + that can be expressed as a function of location. +

+

Cliffs

+

+ We've added a 'continuity' control for cliffs. If you really + like mazes of cliffs, set it to high to reduce the number of gaps in cliff faces. + Or you can turn it way down to make cliffs very rare or be completely absent. +

+

+ +
Changing cliff frequency and continuity. Since cliffs are based on elevation, + you'll have to turn frequency way up if you want lots of layers + even near the starting lake. +

+

Biome Debugging

+

+ Tile placement is based on a range of humidity and 'aux' values + (humidity and aux being properties that vary at different points across the world) + that are suitable for each type of tile. For example: grass is only placed in + places with relatively high humidity, and desert (not to be confused with plain old sand) + only gets placed where aux is high. We've taken to calling these constraints 'rectangles', + because when you plot each tile's home turf on a chart of humidity and aux, + they are shown as rectangles. +

+

+ It's hard to make sense of the rectangles just by looking at the autoplace code + for each tile, so I wrote a script to chart them. This allowed us to ensure that + they were arranged as we wanted, with no gaps between them, + and with overlap in some cases. +

+

+ +
Rectangles. +

+

+ Having the humidity-aux-tile chart is all well and good, but doesn't tell the whole story, + since tile placement also depends on a noise layer specific to each tile type, + and could also influenced by user-adjustable autoplace controls (e.g. turning the grass slider up). + So to further help us visualize how humidity, aux, tile-specific noise, and + autoplace controls worked together to determine tiles on the map, + there are a couple of alternate humidity and aux generators that simply vary them + linearly from north-south and west-east, respectively. +

+

+ +
Using 'debug-moisture' and 'debug-aux' generators to drive moisture and aux, respectively. +

+

+ This map helped us realize that, rather than having controls + for each different type of tile, it made more sense to just + control moisture and aux (which is called 'terrain type' in the GUI, + because 'aux' doesn't mean anything). +

+

+ +
Sliding the moisture and aux bias sliders to make the world more or less grassy or red-deserty. +

+

+ A pet project of mine has been to + put controls in the map generator GUI so that we could select generators + for various tile properties (temperature, aux, humidity, elevation, etc) at + map-creation time without necessarily needing to involve mods. + This was useful for debugging the biome rectangles, but my ulterior + motive was to, at least in cases where there are multiple options, + show the generator information to players. A couple of reasons for + this: +

+
    +
  • It was already possible for mods to override tile property generators via presets, but + we didn't have a place to show that information in the UI. + So switching between presets could change hidden variables in non-obvious ways + and lead to a lot of confusion.
    • +
  • I had dreams of shipping alternate elevation generators in the base game.
    • +
+

Water Placement

+

+ For 0.16 I attempted to make the shape of continents more interesting. Some people really liked the new terrain, or at least managed to find some settings that made it work for them. Others called it a "swampy mess". A common refrain was that the world was more fun to explore in the 0.12 days. +

+

+ So in 0.17 we're restoring the default elevation generator to one very similar to that used + by 0.12. Which means large, sometimes-connected lakes. +

+

+ The water 'frequency' control was confusing to a lot of people including myself. + It could be interpreted as "how much water", when the actual effect was to inversely + scale both bodies of water and continents, such that higher water frequency actually meant smaller bodies of water. + So for 0.17, the water 'frequency' and 'size' sliders are being replaced with 'scale' and 'coverage', + which do the same thing, but in a hopefully more obvious way. + Larger scale means larger land features, while more coverage means more of the map covered in water. +

+

New Map Types

+

+ In order to ensure a decent starting area, the elevation generator + always makes a plateau there (so you'll never spawn in the middle of + the ocean), and a lake (so you can get a power plant running). + Depending on what's going on outside of that plateau, this sometimes resulted in + a circular ring of cliffs around the starting point, + which looked very artificial, and we wanted to reduce that effect. +

+

+ In the process of solving that problem I created another custom generator for debugging purposes. + This one simply generated that starting area plateau in an endless ocean. + I don't actually remember how this was useful for debugging, but at one point I directed Twinsen to look at it + to illustrate the mechanics behind generating the starting area. +

+

+ The rest of the team liked that setting so much that we're making it a player-selectable option. + So in 0.17 you'll get to pick between the 'Normal' map type, which resembles that from 0.12, + and 'Island', which gives you a single large-ish island at the starting point. + There's a slider to let you change the size of the island(s). +

+

+ +

+

+ Maps with multiple starting points will have multiple islands. +

+

+ +
PvP islands! +

+

+ And speaking of scale sliders, we're expanding their range from ± a factor of 2 (the old 'very low' to 'very high' settings) + to ± a factor of 6 (shown as '17%' to '600%'). Previously the values were stored internally as one of 5 discrete options, + but as the recent terrain generation changes have made actual numeric multipliers more meaningful in most contexts + (e.g. the number of ore patches is directly proportional to the value of the 'frequency' slider, + rather than being just vaguely related to it somehow), + we're switching to storing them as numbers. + This has the side-effect that if you don't mind + editing some JSON, + you'll be able to create maps with values outside the range provided by the GUI sliders. +

+

+ Mods will be able to add their own 'map types' to the map type drop-down, too. If you really liked the shape of landmasses in 0.16 and want to be able to continue creating new maps with it, please let us know on the forum. +

+

High-res accumulators (Ernestas, Albert) +

+

+ +

+

+ The design of the accumulator has been always good. The 4 very visible cylinders, looking like giant batteries, Tesla poles and the electric beams perfectly telegraphed its function in terms of style and readability. That’s why for the high-res conversion we were very careful about keeping this entity as it was. +

+

+ The only thing that was a bit disturbing (for some) are the poles crossing to each other when more than one accumulator is placed in a row. So we decided to fix it (or break it). The rest of the work was making the entity compatible for the actual look of the game. But in essence accumulators are still the same. +

+

+ +

+

+ As always, let us know what you think on our forum. +

+ + +
+ + + + + + +
+
\ No newline at end of file diff --git a/test/test-pages/keep-tabular-data/source.html b/test/test-pages/keep-tabular-data/source.html new file mode 100644 index 0000000..8e15377 --- /dev/null +++ b/test/test-pages/keep-tabular-data/source.html @@ -0,0 +1,740 @@ + + + + Friday Facts #282 - 0.17 in sight | Factorio + + + + + + + + + + + + + + + +
+ +
+
+ +
+
+
+ log in + sign up +
+
+ +
+
+
+
+
+ +
+
+

+ Friday Facts #282 - 0.17 in sight +

+
Posted by kovarex, TOGos, Ernestas, Albert on 2019-02-15, all posts
+

+

+

The release plan (kovarex) +

+

This week was the time to close and finish all the things that will go to 0.17.0.

+

Not all of the things that we originally planned to be done were done (surprise), but we just left any non-essential stuff for later so we won't postpone the release any further. The plan is, that next week will be dedicated to the office playtesting and bugfixing. Many would argue, that we could just release instantly and let the players find the bugs for us, but we want to fix the most obvious problems in-house to avoid too many duplicate bug reports and chaos after the release. Also, some potential bugs, like save corruptions, are much more easily worked on in-house.

+

+ If the playtesting goes well, we will let you know next Friday, and if it is the case, we will aim to release the week starting 25th February. +

+

After release plan

+

+ Since there are a lot of things we would like to do before we can call 0.17 good enough, we will simply push new things into the 0.17 releases as time goes on. Even if 0.17 becomes stable in a reasonable time, we would still push things on top of it. We can still make experimental/stable version numbers inside 0.17. Most of the things shouldn't be big enough to make the game generally unstable. I've heard countless times a proposal to make small frequent releases of what have we added, this will probably be reality after 0.17 for some time. +

+

+ The smaller releases will contain mainly: +

+
    +
  • Final looks and behaviour of new GUI screens as they will be finished.
    • +
  • New graphics.
    • +
  • New sounds and sound system tweaks.
    • +
  • Mini tutorial additions and tweaks.
    • +
+

+

+ This is actually quite a large change to our procedures, and there are many ways we will be trying to maximize the effectiveness of smaller and more regular content updates. +

+

The GUI progress (kovarex) +

+

+ There are several GUI screens that are finished. Others (most of them) are just left there as they are in 0.16. They are a combination of the new GUI styles and old ones. They sometimes look funny and out of place, but they should be functional. +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
General UXUX draftUX reviewUI mockupUI reviewImplementation draftImplementation reviewFinal review
Load map
Save map
Graphics settings
Control settings
Sound settings
Interface settings
Other settings
Map generator
Quick bar Twinsen
Train GUI kovarex
Technology GUI Oxyd
Technology tooltip Oxyd
Blueprint library kovarex
Shortcut bar Oxyd
Character screen Dominik
Help overlay kovarex
Manage/Install mods Rseding
Recipe/item/Entity tooltip Twinsen
Chat icon selector ?
New game ?
Menu structure ?
Main screen chat ?
Recipe explorer ?
+ * Newly finished things since the last update in FFF-277. +

Blueprint library

+

+ The blueprint library changes have been split into several steps. The reason is, that there was a big motivation to do the integration with the new quickbar (final version introduced in FFF-278) in time for 0.17.0, while the other changes can be done after. The thing with the quickbar is, that it is quite a big change to one of the most used tools in the game and people generally don't like change even when it is for the better. To minimize the hate of the change, we need to "sell it properly". By that, we should provide as many of the positive aspects of the new quickbar at the time of its introduction. +

+

+ So the change that is already implemented and working for 0.17 is the ability to put blueprints/books into the quick bar in a way that the quick bar is linked directly to the blueprint library window, so you don't need to have the physical blueprint items in your inventory. The other change is, that picking a blueprint from the blueprint library and then pressing Q will just dismiss it, instead of silently pushing it to your inventory. This works the same as the clipboard described in FFF-255. You can still explicitly insert the blueprint from the library to an inventory slot, but if you just pick it, use it, and press Q, it goes away. +

+

+ In addition to this, other changes related to the blueprint library will follow soon after 0.17.0. The first thing is the change of how the GUI looks: +

+

+ +

+

+ We will also allow to switch between grid and list view. It mainly provides a way to nicely see the longer names of the blueprint. We noticed that players try to put a large amount of info about a blueprint in its name, so we are planning to add a possibility to write a textual description of the blueprint. +

+

+ +

+

+ The last big change is to allow to put blueprint books into blueprint books, allowing better organisation. Basically like a directory structure. Whenever a blueprint/book is opened, we plan to show its current location, so the player knows exactly what is going on. +

+

+ +

+

The hand

+

+ Has it ever happened to you, that you have robots trying to put things into your full inventory, while you pick an item from it to build something, and then you just can't put it back, as the diligent robots just filled the last slot in your inventory by whatever they are trying to give to you? Wood from tree removal is the most frequent thing in my case. +

+

+ This was annoying in 0.16 from time to time, but with the new quickbar, it started to happen even more, as now, you have only one inventory, and no reserved slots in the quickbar. To solve that, we just extended the "principal" of the hand. When you pick something from the inventory, the hand icon appears on the slot. As long as you hold the thing in your cursor, the hand stays there, and prevents other things from being inserted there. This way, you should always be able to return the currently selected item into your inventory as long as you didn't get it from external source like a chest. +

+

+
+ The hand is protecting the slot from the robots. +

+

Terrain generation updates (TOGoS) +

+

Everyone has different opinions about what makes a good Factorio world. + I've been working on several changes for 0.17, but the overarching theme + has been to make the map generator options screen more intuitive + and more powerful.

+

+ This was talked about somewhat in an earlier FFF (FFF-258) regarding ore placement, + but since then we found more stuff to fix. +

+

Biter Bases

+

+ In 0.16, the size control for biter bases didn't have much effect. + The frequency control changed the frequency, but that also decreased the size of bases, + which wasn't generally what people wanted. +

+

+ For 0.17 we've reworked biter placement using a system similar to that with which we got resource placement under control. The size and frequency controls now act more like most people would expect, with frequency increasing the number of bases, and size changing the size of each base. +

+

+ +
New preview UI showing the effects of enemy base controls. + In reality the preview takes a couple seconds to regenerate after every change, + but the regeneration part is skipped in this animation to clearly show the effects of the controls. +

+

+ If you don't like the relatively uniform-across-the-world placement of biters, + there are changes under the hood to make it easier for modders to do something different. + Placement is now based on NamedNoiseExpressions "enemy-base-frequency" and "enemy-base-radius", which in turn reference "enemy-base-intensity". + By overriding any of those, a modder could easily create a map where biters are found only at high elevations, + or only near water, or correlate enemy placement with that of resources, or any other thing + that can be expressed as a function of location. +

+

Cliffs

+

+ We've added a 'continuity' control for cliffs. If you really + like mazes of cliffs, set it to high to reduce the number of gaps in cliff faces. + Or you can turn it way down to make cliffs very rare or be completely absent. +

+

+ +
Changing cliff frequency and continuity. Since cliffs are based on elevation, + you'll have to turn frequency way up if you want lots of layers + even near the starting lake. +

+

Biome Debugging

+

+ Tile placement is based on a range of humidity and 'aux' values + (humidity and aux being properties that vary at different points across the world) + that are suitable for each type of tile. For example: grass is only placed in + places with relatively high humidity, and desert (not to be confused with plain old sand) + only gets placed where aux is high. We've taken to calling these constraints 'rectangles', + because when you plot each tile's home turf on a chart of humidity and aux, + they are shown as rectangles. +

+

+ It's hard to make sense of the rectangles just by looking at the autoplace code + for each tile, so I wrote a script to chart them. This allowed us to ensure that + they were arranged as we wanted, with no gaps between them, + and with overlap in some cases. +

+

+ +
Rectangles. +

+

+ Having the humidity-aux-tile chart is all well and good, but doesn't tell the whole story, + since tile placement also depends on a noise layer specific to each tile type, + and could also influenced by user-adjustable autoplace controls (e.g. turning the grass slider up). + So to further help us visualize how humidity, aux, tile-specific noise, and + autoplace controls worked together to determine tiles on the map, + there are a couple of alternate humidity and aux generators that simply vary them + linearly from north-south and west-east, respectively. +

+

+ +
Using 'debug-moisture' and 'debug-aux' generators to drive moisture and aux, respectively. +

+

+ This map helped us realize that, rather than having controls + for each different type of tile, it made more sense to just + control moisture and aux (which is called 'terrain type' in the GUI, + because 'aux' doesn't mean anything). +

+

+ +
Sliding the moisture and aux bias sliders to make the world more or less grassy or red-deserty. +

+

+ A pet project of mine has been to + put controls in the map generator GUI so that we could select generators + for various tile properties (temperature, aux, humidity, elevation, etc) at + map-creation time without necessarily needing to involve mods. + This was useful for debugging the biome rectangles, but my ulterior + motive was to, at least in cases where there are multiple options, + show the generator information to players. A couple of reasons for + this: +

+
    +
  • It was already possible for mods to override tile property generators via presets, but + we didn't have a place to show that information in the UI. + So switching between presets could change hidden variables in non-obvious ways + and lead to a lot of confusion.
    • +
  • I had dreams of shipping alternate elevation generators in the base game.
    • +
+

Water Placement

+

+ For 0.16 I attempted to make the shape of continents more interesting. Some people really liked the new terrain, or at least managed to find some settings that made it work for them. Others called it a "swampy mess". A common refrain was that the world was more fun to explore in the 0.12 days. +

+

+ So in 0.17 we're restoring the default elevation generator to one very similar to that used + by 0.12. Which means large, sometimes-connected lakes. +

+

+ The water 'frequency' control was confusing to a lot of people including myself. + It could be interpreted as "how much water", when the actual effect was to inversely + scale both bodies of water and continents, such that higher water frequency actually meant smaller bodies of water. + So for 0.17, the water 'frequency' and 'size' sliders are being replaced with 'scale' and 'coverage', + which do the same thing, but in a hopefully more obvious way. + Larger scale means larger land features, while more coverage means more of the map covered in water. +

+

New Map Types

+

+ In order to ensure a decent starting area, the elevation generator + always makes a plateau there (so you'll never spawn in the middle of + the ocean), and a lake (so you can get a power plant running). + Depending on what's going on outside of that plateau, this sometimes resulted in + a circular ring of cliffs around the starting point, + which looked very artificial, and we wanted to reduce that effect. +

+

+ In the process of solving that problem I created another custom generator for debugging purposes. + This one simply generated that starting area plateau in an endless ocean. + I don't actually remember how this was useful for debugging, but at one point I directed Twinsen to look at it + to illustrate the mechanics behind generating the starting area. +

+

+ The rest of the team liked that setting so much that we're making it a player-selectable option. + So in 0.17 you'll get to pick between the 'Normal' map type, which resembles that from 0.12, + and 'Island', which gives you a single large-ish island at the starting point. + There's a slider to let you change the size of the island(s). +

+

+ +

+

+ Maps with multiple starting points will have multiple islands. +

+

+ +
PvP islands! +

+

+ And speaking of scale sliders, we're expanding their range from ± a factor of 2 (the old 'very low' to 'very high' settings) + to ± a factor of 6 (shown as '17%' to '600%'). Previously the values were stored internally as one of 5 discrete options, + but as the recent terrain generation changes have made actual numeric multipliers more meaningful in most contexts + (e.g. the number of ore patches is directly proportional to the value of the 'frequency' slider, + rather than being just vaguely related to it somehow), + we're switching to storing them as numbers. + This has the side-effect that if you don't mind + editing some JSON, + you'll be able to create maps with values outside the range provided by the GUI sliders. +

+

+ Mods will be able to add their own 'map types' to the map type drop-down, too. If you really liked the shape of landmasses in 0.16 and want to be able to continue creating new maps with it, please let us know on the forum. +

+

High-res accumulators (Ernestas, Albert) +

+

+ +

+

+ The design of the accumulator has been always good. The 4 very visible cylinders, looking like giant batteries, Tesla poles and the electric beams perfectly telegraphed its function in terms of style and readability. That’s why for the high-res conversion we were very careful about keeping this entity as it was. +

+

+ The only thing that was a bit disturbing (for some) are the poles crossing to each other when more than one accumulator is placed in a row. So we decided to fix it (or break it). The rest of the work was making the entity compatible for the actual look of the game. But in essence accumulators are still the same. +

+

+ +

+

+ As always, let us know what you think on our forum. +

+

+
+
+ Copyright © 2015 - 2019 Wube Software - all rights reserved. +
+
+ Terms of Service + | + Privacy + | + Imprint + | + Presskit + | + FAQ + | + RSS + | + + Jobs + +
+
+
+ + + + + + +
+
+ + + + + \ No newline at end of file diff --git a/test/test-pages/lifehacker-post-comment-load/expected-images.json b/test/test-pages/lifehacker-post-comment-load/expected-images.json index b46843d..1efdcbd 100644 --- a/test/test-pages/lifehacker-post-comment-load/expected-images.json +++ b/test/test-pages/lifehacker-post-comment-load/expected-images.json @@ -1,11 +1,11 @@ -{ - "0": "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--hqqO9fze--\/n1s6c2m6kc07iqdyllj6.jpg", - "2": "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--QeUTCiuW--\/o4dpyrcbiqyfrc3bxx6p.jpg", - "3": "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--mlR3Ku0_--\/xrhkwleyurcizy4akiae.jpg", - "4": "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--Tacb0tyW--\/qodag11euf2npkawkn9v.jpg", - "5": "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--x9hLbIKJ--\/imfc9ybqfw0jmztbhfrh.jpg", - "6": "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--6NwBgQLy--\/afy7n45jfvsjdmmhonct.jpg", - "7": "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--ciqk42G0--\/s3pq8vjrvyjgne4lfsod.jpg", - "8": "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--mtob1sjR--\/y2ldv5eufb3jcrtfouye.jpg", - "9": "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--4Ajak63w--\/atb9qm07fvvg7hqkumkw.jpg" -} \ No newline at end of file +[ + "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--hqqO9fze--\/n1s6c2m6kc07iqdyllj6.jpg", + "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--QeUTCiuW--\/o4dpyrcbiqyfrc3bxx6p.jpg", + "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--mlR3Ku0_--\/xrhkwleyurcizy4akiae.jpg", + "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--Tacb0tyW--\/qodag11euf2npkawkn9v.jpg", + "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--x9hLbIKJ--\/imfc9ybqfw0jmztbhfrh.jpg", + "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--6NwBgQLy--\/afy7n45jfvsjdmmhonct.jpg", + "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--ciqk42G0--\/s3pq8vjrvyjgne4lfsod.jpg", + "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--mtob1sjR--\/y2ldv5eufb3jcrtfouye.jpg", + "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--4Ajak63w--\/atb9qm07fvvg7hqkumkw.jpg" +] \ No newline at end of file diff --git a/test/test-pages/lifehacker-working/expected-images.json b/test/test-pages/lifehacker-working/expected-images.json index b46843d..1efdcbd 100644 --- a/test/test-pages/lifehacker-working/expected-images.json +++ b/test/test-pages/lifehacker-working/expected-images.json @@ -1,11 +1,11 @@ -{ - "0": "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--hqqO9fze--\/n1s6c2m6kc07iqdyllj6.jpg", - "2": "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--QeUTCiuW--\/o4dpyrcbiqyfrc3bxx6p.jpg", - "3": "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--mlR3Ku0_--\/xrhkwleyurcizy4akiae.jpg", - "4": "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--Tacb0tyW--\/qodag11euf2npkawkn9v.jpg", - "5": "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--x9hLbIKJ--\/imfc9ybqfw0jmztbhfrh.jpg", - "6": "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--6NwBgQLy--\/afy7n45jfvsjdmmhonct.jpg", - "7": "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--ciqk42G0--\/s3pq8vjrvyjgne4lfsod.jpg", - "8": "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--mtob1sjR--\/y2ldv5eufb3jcrtfouye.jpg", - "9": "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--4Ajak63w--\/atb9qm07fvvg7hqkumkw.jpg" -} \ No newline at end of file +[ + "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--hqqO9fze--\/n1s6c2m6kc07iqdyllj6.jpg", + "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--QeUTCiuW--\/o4dpyrcbiqyfrc3bxx6p.jpg", + "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--mlR3Ku0_--\/xrhkwleyurcizy4akiae.jpg", + "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--Tacb0tyW--\/qodag11euf2npkawkn9v.jpg", + "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--x9hLbIKJ--\/imfc9ybqfw0jmztbhfrh.jpg", + "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--6NwBgQLy--\/afy7n45jfvsjdmmhonct.jpg", + "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--ciqk42G0--\/s3pq8vjrvyjgne4lfsod.jpg", + "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--mtob1sjR--\/y2ldv5eufb3jcrtfouye.jpg", + "http:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/s--4Ajak63w--\/atb9qm07fvvg7hqkumkw.jpg" +] \ No newline at end of file diff --git a/test/test-pages/nytimes-3/expected-images.json b/test/test-pages/nytimes-3/expected-images.json new file mode 100644 index 0000000..7f4e9ba --- /dev/null +++ b/test/test-pages/nytimes-3/expected-images.json @@ -0,0 +1,6 @@ +[ + "https:\/\/static01.nyt.com\/images\/2019\/02\/21\/nyregion\/21winterutilities1\/00winterutilities1-videoSixteenByNineJumbo1600.jpg", + "https:\/\/static01.nyt.com\/images\/2019\/02\/21\/nyregion\/21winterutilities1\/merlin_150498339_cf9085e5-9756-4169-a5a5-5b516316a3fa-articleLarge.jpg?quality=75&auto=webp&disable=upscale", + "https:\/\/static01.nyt.com\/images\/2018\/02\/20\/multimedia\/author-corey-kilgannon\/author-corey-kilgannon-thumbLarge.jpg", + "https:\/\/static01.nyt.com\/images\/2019\/02\/21\/nyregion\/21winterutilitiesOAK15\/merlin_56025490_f9412a36-eeb9-4a10-a41e-f324eb7a3248-articleLarge.jpg?quality=75&auto=webp&disable=upscale" +] \ No newline at end of file diff --git a/test/test-pages/nytimes-3/expected-metadata.json b/test/test-pages/nytimes-3/expected-metadata.json new file mode 100644 index 0000000..6b5f0b3 --- /dev/null +++ b/test/test-pages/nytimes-3/expected-metadata.json @@ -0,0 +1,8 @@ +{ + "Author": "By Corey Kilgannon", + "Direction": null, + "Excerpt": "New York\u2019s aging below-street infrastructure is tough to maintain, and the corrosive rock salt and \u201cfreeze-thaw\u201d cycles of winter make it even worse.", + "Image": "https:\/\/static01.nyt.com\/images\/2019\/02\/21\/nyregion\/21winterutilities1\/00winterutilities1-videoSixteenByNineJumbo1600.jpg", + "Title": "Manhole Fires and Burst Pipes: How Winter Wreaks Havoc on What\u2019s Underneath N.Y.C.", + "SiteName": null +} \ No newline at end of file diff --git a/test/test-pages/nytimes-3/expected.html b/test/test-pages/nytimes-3/expected.html new file mode 100644 index 0000000..9240af4 --- /dev/null +++ b/test/test-pages/nytimes-3/expected.html @@ -0,0 +1,335 @@ +
+ +
+ + +

+ New York’s aging below-street infrastructure is tough to maintain, and the corrosive rock salt and “freeze-thaw” cycles of winter make it even worse. +

+
+ +
+
+

Image +

+
+ A Con Edison worker repairing underground cables this month in Flushing, Queens. The likely source of the problem was water and rock salt that had seeped underground.CreditCreditChang W. Lee/The New York Times +
+
+
+
+
+
+
+

Corey Kilgannon +

+ +
+
    +
  • + +
    • +
  • + +
    • +
+
+
+
+
+
+
+

+ [What you need to know to start the day: Get New York Today in your inbox.] +

+

+ A series of recent manhole fires in the heart of Manhattan forced the evacuation of several theaters and was a stark reminder that the subway is not the only creaky infrastructure beneath the streets of New York City. +

+

+ Underground lies a chaotic assemblage of utilities that, much like the subway, are lifelines for the city: a sprawling tangle of water mains, power cables, gas and steam lines, telecom wires and sewers. +

+

+ The city has one of the oldest and largest networks of subterranean infrastructure in the world, with some portions dating more than a century and prone to leaks and cracks. +

+

+ And winter — from the corrosive rock salt used on streets and sidewalks to “freeze-thaw” cycles that weaken pipes — makes infrastructure problems even worse. +

+
+ +
+
+
+

+ In the late 1800s, many of the city’s overhead utilities were buried to lessen the exposure to winter weather. “People think it’s all protected and safe, but it’s really not,” said Patrick McHugh, vice president of electrical engineering and planning for Con Edison, which maintains about 90,000 miles of underground cable in the city. +

+

+ “You have water, sewage, electricity and gas down there, and people don’t appreciate the effort that goes into keeping all that working,” he added. +

+
+ +
+
+ +
+
+

Image +

+
+ In the late 1800s, overhead utilities were buried to lessen the exposure to winter weather.CreditKirsten Luce for The New York Times +
+
+
+
+
+
+ +

+ When rock salt melts ice, and the water seeps down manholes and into electrical units, it can set off fires and explosions strong enough to pop a 300-pound manhole cover five stories into the air. +

+

+ For days after a storm, Con Edison officials say, they often deal with scores of electrical fires caused by the rock salt eating away at electrical cable insulation. The wet salt can create sparking that burns the insulation, producing both fire and gases that can combust and pop the manhole lids. +

+

+ To alleviate the threat, the officials said, the utility switched most of its manhole covers to vented ones that allow gases to escape, “so they cannot form a combustible amount,” Mr. McHugh said. +

+
+ +
+
+
+

+ “It also lets smoke escape, which can tip off the public to notify the authorities,” he added. +

+

+ Winter can also bring an increase in gas-line breakages. Con Edison, which maintains 4,300 miles of gas mains in and around New York City, records about 500 leaks — most of them nonemergencies — in a typical month, but many more in winter. +

+

+ Even this past January, which was unseasonably mild, there were 750 leaks, Con Edison officials said. +

+
+ +
+
+ +
+
+
+

Image

+ +
+
+
+ There are typically between 400 and 600 water main breaks each year in New York City, an official said.CreditMichael Appleton for The New York Times +
+
+
+
+
+
+ +

+ The extreme temperature swings that many researchers link to climate change are adding to the challenges of winter. +

+

+ Officials monitor weather forecasts closely for freeze-thaw cycles, when they put extra repair crews on call. +

+

+ During a polar vortex in late January, for instance, single-digit temperatures in the city quickly ballooned into the 50s. The thaw, much welcomed by many New Yorkers, worried Tasos Georgelis, deputy commissioner for water and sewer operations at the Department of Environmental Protection, which operates the city’s water system. +

+

+ “When you get a freeze and a thaw, the ground around the water mains expands and contracts, and puts external pressure on the pipes,” Mr. Georgelis said. +

+

+ Along the city’s roughly 6,500 miles of water mains, there are typically between 400 and 600 breaks a year, he added. The majority occur in winter, when the cold can make older cast-iron mains brittle. +

+
+ +
+
+
+

+ Environmental Protection officials said the department repaired 75 water-main breaks in January, including one in Lower Manhattan that disrupted rush-hour subway service and another on the West Side that snarled traffic and left nearby buildings without water for hours. +

+

+ The city’s 7,500 miles of sewer lines are less affected by cold weather because they are generally buried deeper than other utilities, below the frost line, agency officials said. +

+
+ +
+
+ +
+
+
+

Image

+ +
+
+
+ In 1978, a water main break caused severe flooding in Bushwick, Brooklyn.CreditFred R. Conrad/The New York Times +
+
+
+
+
+
+ +

+ Upgrading the city’s below-street utilities is a slow, painstaking process, “because you have such a fixed-in-place system,” said Rae Zimmerman, a research professor of planning and public administration at New York University. +

+

+ But there is progress. Con Edison officials said they had begun replacing the city’s nearly 1,600 miles of natural gas lines — which were made of either cast iron or unprotected steel — with plastic piping. The plastic is less susceptible to corrosion, cracks and leaks, said the officials, who added that they were swapping about 100 miles of line each year. +

+

+ The city is also replacing older, leak-prone water and sewer mains. +

+

+ Some pipes that are more than a century old hold up because they were built with a thicker grade of cast iron, according to Environmental Protection Department officials. For less healthy ones, the agency has invested more than $1 billion in the past five years — with an additional $1.4 billion budgeted over the next five years — for upgrades and replacements. New pipes will be made of a more durable, graphite-rich cast iron known as ductile iron. +

+
+ +
+
+ +
+
+
+

Image

+ +
+
+
+ Matt Cruz snowboarded through Manhattan’s Lower East Side after a snowstorm in 2016 left the streets coated in slush and rock salt.CreditHiroko Masuike/The New York Times +
+
+
+
+
+
+ +

+ Of course, winter also poses problems aboveground. Most nonemergency repair and construction work involving concrete is halted because concrete and some types of dirt, used to fill in trenches, freeze in colder temperatures, said Ian Michaels, a spokesman for the city’s Department of Design and Construction. +

+

+ Digging by hand is also a challenge in frozen ground, so many excavations that are close to pipes and other utilities are put off, Mr. Michaels said. +

+
+ +
+
+
+

+ And asphalt is harder to obtain because it must be kept and transported at high temperatures, he added. +

+

+ In the extreme cold, city officials will not risk shutting down water mains for construction because spillage into the street could freeze, Mr. Michaels said. He added that stopping the water flow could freeze the private water-service connections that branch off the mains, he said. +

+

+ Even the basic task of locating utilities under the street can be complicated because infrastructure has been added piecemeal over the decades. +

+
+ +
+
+ +
+
+
+

Image

+ +
+
+
+ A water main break in Manhattan in 2014. “When you get a freeze and a thaw, the ground around the water mains expands and contracts, and puts external pressure on the pipes,” said Tasos Georgelis of the city's Department of Environmental Protection.CreditÃngel Franco/The New York Times +
+
+
+
+
+
+ +

+ Street surfaces are affected by winter weather, too: Last year, the city filled 255,904 potholes. +

+

+ And should anyone forget that filling potholes, like snow removal, is a sacred staple of constituent services, transportation officials have compiled the number of potholes the city has filled — more than 1,786,300 — since Mayor Bill de Blasio took office in 2014. +

+

+ Potholes form when water and salt seep into cracks, freeze and expand, creating a larger crevice, said Joe Carbone, who works for the Transportation Department, where he is known as the pothole chief. +

+

+ Simply put, more freeze-thaw cycles result in more potholes, he said. Currently, the department has 25 crews repairing potholes. During peak pothole-repair season in early March, that number can expand to more than 60. +

+
+ +
+
+

+ Still, the department is continually resurfacing the city’s more than 6,000 miles of streets and 19,000 lane miles. Each year, agency officials said, it uses more than one million tons of asphalt to repave more than 1,300 lane-miles of street. +

+ +
+
+ +
+
+
+

Image

+ +
+
+
+ Workers learning how to fix water main breaks at a training center in Queens.CreditChang W. Lee/The New York Times +
+
+
+
+
+
+ +

+ Of the 400 city laborers who work on water mains, many learn the finer points of leak repair at a training center in Queens, where underground pipes are made to spring leaks for repair drills. +

+

+ Workers from the Department of Environmental Protection recently gathered around a muddy hole as a co-worker, Nehemiah Dejesus, scrambled to apply a stainless-steel repair clamp around a cracked segment that was spewing water. +

+

+ “Don’t get nervous,” instructed Milton Velez, the agency’s district supervisor for Queens. +

+

+ “I’m not,” Mr. Dejesus said as he secured the clamp and stopped the leak. “It’s ‘Showtime at the Apollo.’” +

+
+ +
+ + +
+
+ +
+
+

+ Corey Kilgannon is a Metro reporter covering news and human interest stories. His writes the Character Study column in the Sunday Metropolitan section. He was also part of the team that won the 2009 Pulitzer Prize for Breaking News. @coreykilgannon • Facebook +

+
+
+

+ A version of this article appears in print on

, on Page

A

22

of the New York edition

with the headline:

Under the City’s Streets, A Battle Against Winter. Order Reprints | Today’s Paper | Subscribe +

+ +
+ +
\ No newline at end of file diff --git a/test/test-pages/nytimes-3/source.html b/test/test-pages/nytimes-3/source.html new file mode 100644 index 0000000..2936576 --- /dev/null +++ b/test/test-pages/nytimes-3/source.html @@ -0,0 +1,1638 @@ + + + + + + Manhole Fires and Burst Pipes: How Winter Wreaks Havoc on What’s Underneath N.Y.C. - The New York Times + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+
+
+
+
+
+
+
+
+ +
+
+ +
Skip to contentSkip to site index +
+
+ New York +
+
+ + +
+
+ +
+
+ +
+
+
+
+
+
+
+ Log In +
+
+ Today’s Paper +
+
+
+
+
+
+
+ + + + +
+
+
+
+
+ +
+

+ Manhole Fires and Burst Pipes: How Winter Wreaks Havoc on What’s Underneath N.Y.C. +

+
+

+ New York’s aging below-street infrastructure is tough to maintain, and the corrosive rock salt and “freeze-thaw” cycles of winter make it even worse. +

+
+
+
+
+
+ Image +
+
+ A Con Edison worker repairing underground cables this month in Flushing, Queens. The likely source of the problem was water and rock salt that had seeped underground.CreditCreditChang W. Lee/The New York Times +
+
+
+
+
+
+
+
+ Corey Kilgannon +
+
+

+ By Corey Kilgannon +

+
+
+
    +
  • + +
    • +
  • +
    + +
    +
    • +
+
+
+
+
+
+
+

+ [What you need to know to start the day: Get New York Today in your inbox.] +

+

+ A series of recent manhole fires in the heart of Manhattan forced the evacuation of several theaters and was a stark reminder that the subway is not the only creaky infrastructure beneath the streets of New York City. +

+

+ Underground lies a chaotic assemblage of utilities that, much like the subway, are lifelines for the city: a sprawling tangle of water mains, power cables, gas and steam lines, telecom wires and sewers. +

+

+ The city has one of the oldest and largest networks of subterranean infrastructure in the world, with some portions dating more than a century and prone to leaks and cracks. +

+

+ And winter — from the corrosive rock salt used on streets and sidewalks to “freeze-thaw” cycles that weaken pipes — makes infrastructure problems even worse. +

+
+ +
+
+
+

+ In the late 1800s, many of the city’s overhead utilities were buried to lessen the exposure to winter weather. “People think it’s all protected and safe, but it’s really not,” said Patrick McHugh, vice president of electrical engineering and planning for Con Edison, which maintains about 90,000 miles of underground cable in the city. +

+

+ “You have water, sewage, electricity and gas down there, and people don’t appreciate the effort that goes into keeping all that working,” he added. +

+
+ +
+
+
+
+
+
+ Image +
+
+ In the late 1800s, overhead utilities were buried to lessen the exposure to winter weather.CreditKirsten Luce for The New York Times +
+
+
+
+
+
+ +

+ When rock salt melts ice, and the water seeps down manholes and into electrical units, it can set off fires and explosions strong enough to pop a 300-pound manhole cover five stories into the air. +

+

+ For days after a storm, Con Edison officials say, they often deal with scores of electrical fires caused by the rock salt eating away at electrical cable insulation. The wet salt can create sparking that burns the insulation, producing both fire and gases that can combust and pop the manhole lids. +

+

+ To alleviate the threat, the officials said, the utility switched most of its manhole covers to vented ones that allow gases to escape, “so they cannot form a combustible amount,” Mr. McHugh said. +

+
+ +
+
+
+

+ “It also lets smoke escape, which can tip off the public to notify the authorities,” he added. +

+

+ Winter can also bring an increase in gas-line breakages. Con Edison, which maintains 4,300 miles of gas mains in and around New York City, records about 500 leaks — most of them nonemergencies — in a typical month, but many more in winter. +

+

+ Even this past January, which was unseasonably mild, there were 750 leaks, Con Edison officials said. +

+
+ +
+
+
+
+
+
+ Image +
+
+
+
+
+ There are typically between 400 and 600 water main breaks each year in New York City, an official said.CreditMichael Appleton for The New York Times +
+
+
+
+
+
+ +

+ The extreme temperature swings that many researchers link to climate change are adding to the challenges of winter. +

+

+ Officials monitor weather forecasts closely for freeze-thaw cycles, when they put extra repair crews on call. +

+

+ During a polar vortex in late January, for instance, single-digit temperatures in the city quickly ballooned into the 50s. The thaw, much welcomed by many New Yorkers, worried Tasos Georgelis, deputy commissioner for water and sewer operations at the Department of Environmental Protection, which operates the city’s water system. +

+

+ “When you get a freeze and a thaw, the ground around the water mains expands and contracts, and puts external pressure on the pipes,” Mr. Georgelis said. +

+

+ Along the city’s roughly 6,500 miles of water mains, there are typically between 400 and 600 breaks a year, he added. The majority occur in winter, when the cold can make older cast-iron mains brittle. +

+
+ +
+
+
+

+ Environmental Protection officials said the department repaired 75 water-main breaks in January, including one in Lower Manhattan that disrupted rush-hour subway service and another on the West Side that snarled traffic and left nearby buildings without water for hours. +

+

+ The city’s 7,500 miles of sewer lines are less affected by cold weather because they are generally buried deeper than other utilities, below the frost line, agency officials said. +

+
+ +
+
+
+
+
+
+ Image +
+
+
+
+
+ In 1978, a water main break caused severe flooding in Bushwick, Brooklyn.CreditFred R. Conrad/The New York Times +
+
+
+
+
+
+ +

+ Upgrading the city’s below-street utilities is a slow, painstaking process, “because you have such a fixed-in-place system,” said Rae Zimmerman, a research professor of planning and public administration at New York University. +

+

+ But there is progress. Con Edison officials said they had begun replacing the city’s nearly 1,600 miles of natural gas lines — which were made of either cast iron or unprotected steel — with plastic piping. The plastic is less susceptible to corrosion, cracks and leaks, said the officials, who added that they were swapping about 100 miles of line each year. +

+

+ The city is also replacing older, leak-prone water and sewer mains. +

+

+ Some pipes that are more than a century old hold up because they were built with a thicker grade of cast iron, according to Environmental Protection Department officials. For less healthy ones, the agency has invested more than $1 billion in the past five years — with an additional $1.4 billion budgeted over the next five years — for upgrades and replacements. New pipes will be made of a more durable, graphite-rich cast iron known as ductile iron. +

+
+ +
+
+
+
+
+
+ Image +
+
+
+
+
+ Matt Cruz snowboarded through Manhattan’s Lower East Side after a snowstorm in 2016 left the streets coated in slush and rock salt.CreditHiroko Masuike/The New York Times +
+
+
+
+
+
+ +

+ Of course, winter also poses problems aboveground. Most nonemergency repair and construction work involving concrete is halted because concrete and some types of dirt, used to fill in trenches, freeze in colder temperatures, said Ian Michaels, a spokesman for the city’s Department of Design and Construction. +

+

+ Digging by hand is also a challenge in frozen ground, so many excavations that are close to pipes and other utilities are put off, Mr. Michaels said. +

+
+ +
+
+
+

+ And asphalt is harder to obtain because it must be kept and transported at high temperatures, he added. +

+

+ In the extreme cold, city officials will not risk shutting down water mains for construction because spillage into the street could freeze, Mr. Michaels said. He added that stopping the water flow could freeze the private water-service connections that branch off the mains, he said. +

+

+ Even the basic task of locating utilities under the street can be complicated because infrastructure has been added piecemeal over the decades. +

+
+ +
+
+
+
+
+
+ Image +
+
+
+
+
+ A water main break in Manhattan in 2014. “When you get a freeze and a thaw, the ground around the water mains expands and contracts, and puts external pressure on the pipes,” said Tasos Georgelis of the city's Department of Environmental Protection.CreditÃngel Franco/The New York Times +
+
+
+
+
+
+ +

+ Street surfaces are affected by winter weather, too: Last year, the city filled 255,904 potholes. +

+

+ And should anyone forget that filling potholes, like snow removal, is a sacred staple of constituent services, transportation officials have compiled the number of potholes the city has filled — more than 1,786,300 — since Mayor Bill de Blasio took office in 2014. +

+

+ Potholes form when water and salt seep into cracks, freeze and expand, creating a larger crevice, said Joe Carbone, who works for the Transportation Department, where he is known as the pothole chief. +

+

+ Simply put, more freeze-thaw cycles result in more potholes, he said. Currently, the department has 25 crews repairing potholes. During peak pothole-repair season in early March, that number can expand to more than 60. +

+
+ +
+
+
+

+ Still, the department is continually resurfacing the city’s more than 6,000 miles of streets and 19,000 lane miles. Each year, agency officials said, it uses more than one million tons of asphalt to repave more than 1,300 lane-miles of street. +

+
+ +
+
+
+
+
+
+ Image +
+
+
+
+
+ Workers learning how to fix water main breaks at a training center in Queens.CreditChang W. Lee/The New York Times +
+
+
+
+
+
+ +

+ Of the 400 city laborers who work on water mains, many learn the finer points of leak repair at a training center in Queens, where underground pipes are made to spring leaks for repair drills. +

+

+ Workers from the Department of Environmental Protection recently gathered around a muddy hole as a co-worker, Nehemiah Dejesus, scrambled to apply a stainless-steel repair clamp around a cracked segment that was spewing water. +

+

+ “Don’t get nervous,” instructed Milton Velez, the agency’s district supervisor for Queens. +

+

+ “I’m not,” Mr. Dejesus said as he secured the clamp and stopped the leak. “It’s ‘Showtime at the Apollo.’” +

+
+ +
+
+
+
+ +
+
+ Why Are New York City’s Streets Always Under Construction? +
+
+
+ +
+
+
+
+
+
+ +
+
+
+
+
+
+
+

+ Corey Kilgannon is a Metro reporter covering news and human interest stories. His writes the Character Study column in the Sunday Metropolitan section. He was also part of the team that won the 2009 Pulitzer Prize for Breaking News. @coreykilgannon • Facebook +

+
+
+
+
+ A version of this article appears in print on , on Page A22 of the New York edition with the headline: Under the City’s Streets, A Battle Against Winter. Order Reprints | Today’s Paper | Subscribe +
+
+ +
+
+
+
+

+ Advertisement +

+
+
+
+
+
+
+
+
+ +
+
+
+
+
+ + +
+
+
+ + + + + + +
+ +
+ + -- cgit v1.2.3